j3d1/toolshed
2
1
Fork 0
Code Issues 9 Pull requests 1 Projects Releases Packages Wiki Activity

make dataset parsing more robust
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse source
This commit is contained in:
j3d1 2024-03-11 17:37:56 +01:00
parent f2db5d9dad
commit 8cf0897ec5
13 changed files with 337 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

21
backend/configure.py
View file

@ -83,10 +83,12 @@ def configure():
if yesno("Do you want to import all categories, properties and tags contained in this repository?", default=True):
from hostadmin.serializers import CategorySerializer, PropertySerializer, TagSerializer
from hostadmin.models import ImportedIdentifierSets
from hashlib import sha256
if not os.path.exists('shared_data'):
os.mkdir('shared_data')
files = os.listdir('shared_data')
idsets = {}
hashes = {}
for file in files:
if file.endswith('.json'):
name = "git:" + file[:-5]
@ -94,6 +96,8 @@ def configure():
try:
idset = json.load(f)
idsets[name] = idset
f.seek(0)
hashes[name] = sha256(f.read().encode()).hexdigest()
except json.decoder.JSONDecodeError:
print('Error: invalid JSON in file {}'.format(file))
imported_sets = ImportedIdentifierSets.objects.all()
@ -108,9 +112,13 @@ def configure():
unmet_deps = [dep for dep in idset['depends'] if not imported_sets.filter(name=dep).exists()]
if unmet_deps:
if all([dep in idsets.keys() for dep in unmet_deps]):
print('Not all dependencies for {} are imported, postponing'.format(name))
queue.append(name)
continue
if all([dep in queue for dep in unmet_deps]):
print('Not all dependencies for {} are imported, postponing'.format(name))
queue.append(name)
continue
else:
print('Error: unresolvable dependencies for {}: {}'.format(name, unmet_deps))
continue
else:
print('unknown dependencies for {}: {}'.format(name, unmet_deps))
continue
@ -131,10 +139,15 @@ def configure():
serializer = TagSerializer(data=tag)
if serializer.is_valid():
serializer.save(origin=name)
imported_sets.create(name=name)
imported_sets.create(name=name, hash=hashes[name])
except IntegrityError:
print('Error: integrity error while importing {}\n\tmight be cause by name conflicts with existing'
' categories, properties or tags'.format(name))
transaction.set_rollback(True)
continue
except Exception as e:
print('Error: {}'.format(e))
transaction.set_rollback(True)
continue