make dataset parsing more robust

backend/hostadmin/migrations/0003_importedidentifiersets_hash.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.2 on 2024-03-11 15:19
+import os
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('hostadmin', '0002_importedidentifiersets'),
+    ]
+
+    def calculate_hash(apps, schema_editor):
+        from hostadmin.models import ImportedIdentifierSets
+        for identifier_set in ImportedIdentifierSets.objects.all():
+            if not identifier_set.hash:
+                print("update", identifier_set.name)
+                filename = "shared_data/" + identifier_set.name.strip('git:') + ".json"
+                if not os.path.exists(filename):
+                    continue
+                from hashlib import sha256
+                with open(filename, 'r') as file:
+                    data = file.read()
+                    identifier_set.hash = sha256(data.encode()).hexdigest()
+                    identifier_set.save()
+
+    operations = [
+        migrations.AddField(
+            model_name='importedidentifiersets',
+            name='hash',
+            field=models.CharField(blank=True, max_length=255, null=True),
+
+        ),
+        migrations.RunPython(calculate_hash),
+        migrations.AlterField(
+            model_name='importedidentifiersets',
+            name='hash',
+            field=models.CharField(max_length=255, unique=True),
+        ),
+    ]

backend/hostadmin/migrations/0004_alter_importedidentifiersets_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.2 on 2024-03-14 16:33
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hostadmin', '0003_importedidentifiersets_hash'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='importedidentifiersets',
+            options={'verbose_name_plural': 'imported identifier sets'},
+        ),
+    ]