Merge branch 'create-training-app-models' into 'master'

Create new training apps and models, configure admin view

Closes #960

See merge request !1632

arkindex/project/settings.py

@@ -119,6 +119,7 @@ INSTALLED_APPS = [
     'arkindex.documents',
     'arkindex.users',
     'arkindex.dataimport',
+    'arkindex.training',
 ]
 
 MIDDLEWARE = [
@@ -250,6 +251,7 @@ SPECTACULAR_SETTINGS = {
         'ClassificationState': 'arkindex.documents.models.ClassificationState',
         'PonosState': 'ponos.models.State',
         'WorkerVersionState': 'arkindex.dataimport.models.WorkerVersionState',
+        'ModelVersionState': 'arkindex.training.models.ModelVersionState',
     },
     'TAGS': [
         {'name': 'classifications'},

arkindex/training/admin.py

+from django.contrib import admin
+from enumfields.admin import EnumFieldListFilter
+
+from arkindex.training.models import Model, ModelVersion
+
+
+class ModelAdmin(admin.ModelAdmin):
+    list_display = ('name', 'created', )
+    search_fields = ('name', 'description', )
+    fields = ('name', 'description', 'public', 'compatible_workers')
+
+
+class ModelVersionAdmin(admin.ModelAdmin):
+    list_display = ('id', 'model', 'tag', 'size', 'state')
+    list_filter = ('model__name', ('state', EnumFieldListFilter), )
+    fields = ('model', 'parent', 'description', 'state', 'tag', 'hash', 'size', 'configuration',)
+    readonly_fields = ('hash', 'size', )
+
+
+admin.site.register(Model, ModelAdmin)
+admin.site.register(ModelVersion, ModelVersionAdmin)

arkindex/training/migrations/0001_initial.py

+# Generated by Django 4.0.2 on 2022-03-16 09:50
+
+import uuid
+
+import django.db.models.deletion
+import enumfields.fields
+from django.db import migrations, models
+
+import arkindex.project.fields
+import arkindex.training.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('dataimport', '0045_remove_dataimport_best_class'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Model',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=100, unique=True)),
+                ('description', models.TextField(default='')),
+                ('public', models.BooleanField(default=False)),
+                ('compatible_workers', models.ManyToManyField(related_name='models', to='dataimport.Worker')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ModelVersion',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('description', models.TextField(default='')),
+                ('tag', models.CharField(blank=True, max_length=50, null=True)),
+                ('state', enumfields.fields.EnumField(default='created', enum=arkindex.training.models.ModelVersionState, max_length=10)),
+                ('hash', arkindex.project.fields.MD5HashField(max_length=32)),
+                ('size', models.PositiveIntegerField(help_text='file size in bytes')),
+                ('configuration', models.JSONField(default=dict)),
+                ('model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='training.model')),
+                ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='training.modelversion')),
+            ],
+            options={
+                'unique_together': {('model', 'tag')},
+            },
+        ),
+    ]

arkindex/training/models.py

+from django.db import models
+from enumfields import Enum, EnumField
+
+from arkindex.project.fields import MD5HashField
+from arkindex.project.models import IndexableModel
+
+
+class Model(IndexableModel):
+    """
+    An evolving Machine Learning model
+    """
+    # Name of the model, unique
+    name = models.CharField(max_length=100, unique=True)
+
+    description = models.TextField(default="")
+
+    public = models.BooleanField(default=False)
+
+    # Link to the workers that are able to use this model
+    compatible_workers = models.ManyToManyField('dataimport.Worker', related_name='models')
+
+    def __str__(self):
+        return self.name
+
+
+class ModelVersionState(Enum):
+    """
+    State of the model Version, available meaning checked by the backend
+    """
+    Created = 'created'
+    Available = 'available'
+    Error = 'error'
+
+
+class ModelVersion(IndexableModel):
+    """
+    A specific Model version
+    """
+    model = models.ForeignKey('training.Model', related_name='versions', on_delete=models.CASCADE)
+
+    parent = models.ForeignKey('self', related_name='children', null=True, blank=True, on_delete=models.CASCADE)
+
+    description = models.TextField(default="")
+
+    tag = models.CharField(null=True, max_length=50, blank=True)
+
+    state = EnumField(ModelVersionState, default=ModelVersionState.Created)
+
+    # Hash of the archive
+    hash = MD5HashField()
+
+    # Size of the archive
+    size = models.PositiveIntegerField(help_text='file size in bytes')
+
+    # Store dictionary of paramseters given by the ML developer
+    configuration = models.JSONField(default=dict)
+
+    class Meta:
+        unique_together = (
+            ('model', 'tag'),
+        )