diff --git a/arkindex/documents/migrations/0001_initial.py b/arkindex/documents/migrations/0001_initial.py
index 5d631e8b42b3516677b14075de5414e6cf0035cd..93e291d49f8c948473df6459e5b7275e34640bdc 100644
--- a/arkindex/documents/migrations/0001_initial.py
+++ b/arkindex/documents/migrations/0001_initial.py
@@ -1,44 +1,27 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import uuid
import django.contrib.postgres.fields.hstore
+import django.core.validators
import django.db.models.deletion
import enumfields.fields
from django.db import migrations, models
-from enumfields import Enum
import arkindex.documents.dates
import arkindex.documents.models
+import arkindex.project.aws
import arkindex.project.fields
-class OldEntityType(Enum):
- Person = 'person'
- Location = 'location'
- Subject = 'subject'
- Organization = 'organization'
- Misc = 'misc'
- Number = 'number'
- Date = 'date'
-
-
class Migration(migrations.Migration):
initial = True
dependencies = [
- ('images', '0001_initial'),
- ('process', '0001_initial'),
]
operations = [
- # Leave this when resetting migrations to avoid issues with HStoreField
- # django.db.utils.ProgrammingError: type "hstore" does not exist
- migrations.RunSQL(
- "CREATE EXTENSION IF NOT EXISTS hstore",
- reverse_sql=migrations.RunSQL.noop,
- ),
migrations.CreateModel(
name='AllowedMetaData',
fields=[
@@ -62,25 +45,30 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='Corpus',
fields=[
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.CharField(max_length=250)),
('description', models.TextField(default='')),
('public', models.BooleanField(default=False)),
+ ('indexable', models.BooleanField(default=False)),
],
options={
'verbose_name_plural': 'corpora',
},
),
migrations.CreateModel(
- name='DataSource',
+ name='CorpusExport',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('type', enumfields.fields.EnumField(enum=Enum('MLToolType', ''), max_length=10)),
- ('slug', models.CharField(max_length=100)),
- ('name', models.CharField(max_length=100)),
- ('revision', models.CharField(max_length=100)),
- ('internal', models.BooleanField()),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('state', enumfields.fields.EnumField(default='created', enum=arkindex.documents.models.CorpusExportState, max_length=10)),
],
+ options={
+ 'abstract': False,
+ },
+ bases=(arkindex.project.aws.S3FileMixin, models.Model),
),
migrations.CreateModel(
name='Element',
@@ -89,10 +77,11 @@ class Migration(migrations.Migration):
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=250)),
+ ('polygon', arkindex.project.fields.LinearRingField(blank=True, null=True, srid=0)),
+ ('rotation_angle', models.SmallIntegerField(default=0, help_text='Clockwise rotation to apply to the image after cropping, in degrees.', validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(359)])),
+ ('mirrored', models.BooleanField(default=False, help_text='Mirror the image along the vertical axis before rotating.')),
+ ('confidence', models.FloatField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(1)])),
],
- options={
- 'abstract': False,
- },
),
migrations.CreateModel(
name='ElementPath',
@@ -109,8 +98,7 @@ class Migration(migrations.Migration):
('slug', models.SlugField()),
('display_name', models.CharField(max_length=250)),
('folder', models.BooleanField(default=False)),
- ('hidden', models.BooleanField(default=False)),
- ('default_view', models.BooleanField(default=False)),
+ ('indexable', models.BooleanField(default=False)),
],
),
migrations.CreateModel(
@@ -118,141 +106,83 @@ class Migration(migrations.Migration):
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.TextField()),
- ('type', enumfields.fields.EnumField(db_index=True, enum=OldEntityType, max_length=50)),
('metas', django.contrib.postgres.fields.hstore.HStoreField(blank=True, null=True)),
('validated', models.BooleanField(default=False)),
- ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='documents.Corpus')),
],
options={
'verbose_name_plural': 'Entities',
},
- bases=(arkindex.documents.dates.InterpretedDateMixin, models.Model),
),
migrations.CreateModel(
- name='MLClass',
+ name='EntityLink',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=100)),
- ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ml_classes', to='documents.Corpus')),
],
- options={
- 'verbose_name_plural': 'classes',
- 'ordering': ('corpus', 'name'),
- },
),
migrations.CreateModel(
- name='Region',
+ name='EntityRole',
fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('confidence', models.FloatField(blank=True, null=True)),
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('parent_name', models.CharField(max_length=250)),
+ ('child_name', models.CharField(max_length=250)),
],
- options={
- 'default_related_name': 'regions',
- },
),
migrations.CreateModel(
- name='Transcription',
+ name='EntityType',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('type', models.CharField(db_index=True, default='word', max_length=50)),
- ('text', models.TextField(blank=True, null=True)),
- ('score', models.FloatField(blank=True, null=True)),
- ('element', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='documents.Element')),
+ ('name', models.CharField(max_length=250)),
+ ('color', models.CharField(default='ff0000', max_length=6)),
],
),
migrations.CreateModel(
- name='TranscriptionEntity',
+ name='MetaData',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('offset', models.PositiveIntegerField()),
- ('length', models.PositiveIntegerField()),
- ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcription_entities', to='documents.Entity')),
- ('transcription', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcription_entities', to='documents.Transcription')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=250)),
+ ('type', enumfields.fields.EnumField(db_index=True, enum=arkindex.documents.models.MetaType, max_length=50)),
+ ('value', models.TextField()),
],
- ),
- migrations.AddField(
- model_name='transcription',
- name='entities',
- field=models.ManyToManyField(related_name='transcriptions', through='documents.TranscriptionEntity', to='documents.Entity'),
- ),
- migrations.AddField(
- model_name='transcription',
- name='source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='documents.DataSource'),
- ),
- migrations.AddField(
- model_name='transcription',
- name='zone',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='transcriptions', to='images.Zone'),
+ options={
+ 'ordering': ('element', 'name', 'id'),
+ },
+ bases=(arkindex.documents.dates.InterpretedDateMixin, models.Model),
),
migrations.CreateModel(
- name='RegionElement',
+ name='MLClass',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('ordering', models.PositiveIntegerField(default=0)),
- ('element', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='region_elements', to='documents.Element')),
- ('region', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='region_elements', to='documents.Region')),
+ ('name', models.CharField(max_length=1024)),
],
options={
- 'ordering': ('element', 'ordering'),
- 'default_related_name': 'region_elements',
+ 'verbose_name_plural': 'classes',
+ 'ordering': ('corpus', 'name'),
},
),
- migrations.AddField(
- model_name='region',
- name='elements',
- field=models.ManyToManyField(related_name='regions', through='documents.RegionElement', to='documents.Element'),
- ),
- migrations.AddField(
- model_name='region',
- name='ml_class',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='regions', to='documents.MLClass'),
- ),
- migrations.AddField(
- model_name='region',
- name='source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='regions', to='documents.DataSource'),
- ),
- migrations.AddField(
- model_name='region',
- name='zone',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='regions', to='images.Zone'),
- ),
migrations.CreateModel(
- name='MetaData',
+ name='Selection',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=250)),
- ('type', enumfields.fields.EnumField(db_index=True, enum=arkindex.documents.models.MetaType, max_length=50)),
- ('value', models.TextField()),
- ('index', models.PositiveIntegerField(default=0)),
- ('element', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metadatas', to='documents.Element')),
- ('entity', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='metadatas', to='documents.Entity')),
- ('revision', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='process.Revision')),
],
- options={
- 'ordering': ('element', 'name', 'index'),
- },
- bases=(arkindex.documents.dates.InterpretedDateMixin, models.Model),
),
migrations.CreateModel(
- name='EntityRole',
+ name='Transcription',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('parent_name', models.CharField(max_length=250)),
- ('child_name', models.CharField(max_length=250)),
- ('parent_type', enumfields.fields.EnumField(enum=OldEntityType, max_length=50)),
- ('child_type', enumfields.fields.EnumField(enum=OldEntityType, max_length=50)),
- ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='roles', to='documents.Corpus')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('text', models.TextField()),
+ ('orientation', enumfields.fields.EnumField(default='horizontal-lr', enum=arkindex.documents.models.TextOrientation, max_length=50)),
+ ('confidence', models.FloatField(blank=True, null=True)),
],
),
migrations.CreateModel(
- name='EntityLink',
+ name='TranscriptionEntity',
fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('child', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='children', to='documents.Entity')),
- ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='parents', to='documents.Entity')),
- ('role', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='documents.EntityRole')),
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('offset', models.PositiveIntegerField()),
+ ('length', models.PositiveIntegerField(validators=[django.core.validators.MinValueValidator(1)])),
+ ('confidence', models.FloatField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(1)])),
+ ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcription_entities', to='documents.entity')),
+ ('transcription', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcription_entities', to='documents.transcription')),
],
),
]
diff --git a/arkindex/documents/migrations/0002_initial.py b/arkindex/documents/migrations/0002_initial.py
index fdafe7182bb8b496cb121a736ecc629c22c995a7..f094cf105fd74ed61ca18b02b672a95ac94583ff 100644
--- a/arkindex/documents/migrations/0002_initial.py
+++ b/arkindex/documents/migrations/0002_initial.py
@@ -1,8 +1,6 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
-import django.contrib.postgres.indexes
import django.db.models.deletion
-from django.conf import settings
from django.db import migrations, models
@@ -11,164 +9,44 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('process', '0001_initial'),
('documents', '0001_initial'),
- ('images', '0001_initial'),
]
operations = [
migrations.AddField(
- model_name='entity',
- name='moderator',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='entities',
- to=settings.AUTH_USER_MODEL,
- ),
+ model_name='transcriptionentity',
+ name='worker_run',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='transcription_entities', to='process.workerrun'),
),
migrations.AddField(
- model_name='entity',
- name='source',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='entities',
- to='documents.DataSource',
- ),
+ model_name='transcriptionentity',
+ name='worker_version',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='transcription_entities', to='process.workerversion'),
),
migrations.AddField(
- model_name='elementtype',
- name='corpus',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='types',
- to='documents.Corpus',
- ),
- ),
- migrations.AddField(
- model_name='elementpath',
+ model_name='transcription',
name='element',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='paths',
- to='documents.Element',
- ),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='documents.element'),
),
migrations.AddField(
- model_name='element',
- name='corpus',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='elements',
- to='documents.Corpus',
- ),
+ model_name='transcription',
+ name='entities',
+ field=models.ManyToManyField(related_name='transcriptions', through='documents.TranscriptionEntity', to='documents.entity'),
),
migrations.AddField(
- model_name='element',
- name='type',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.PROTECT,
- related_name='elements',
- to='documents.ElementType',
- ),
+ model_name='transcription',
+ name='worker_run',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='transcriptions', to='process.workerrun'),
),
migrations.AddField(
- model_name='element',
- name='zone',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='elements',
- to='images.Zone',
- ),
- ),
- migrations.AlterUniqueTogether(
- name='datasource',
- unique_together={('type', 'slug', 'revision')},
+ model_name='transcription',
+ name='worker_version',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='process.workerversion'),
),
migrations.AddField(
- model_name='classification',
+ model_name='selection',
name='element',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='classifications',
- to='documents.Element',
- ),
- ),
- migrations.AddField(
- model_name='classification',
- name='ml_class',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='classifications',
- to='documents.MLClass',
- ),
- ),
- migrations.AddField(
- model_name='classification',
- name='moderator',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='classifications',
- to=settings.AUTH_USER_MODEL,
- ),
- ),
- migrations.AddField(
- model_name='classification',
- name='source',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='classifications',
- to='documents.DataSource',
- ),
- ),
- migrations.AddField(
- model_name='allowedmetadata',
- name='corpus',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='allowed_metadatas',
- to='documents.Corpus',
- ),
- ),
- migrations.AlterUniqueTogether(
- name='transcriptionentity',
- unique_together={('transcription', 'entity', 'offset', 'length')},
- ),
- migrations.AlterUniqueTogether(
- name='regionelement',
- unique_together={('element', 'region')},
- ),
- migrations.AlterUniqueTogether(
- name='mlclass',
- unique_together={('name', 'corpus')},
- ),
- migrations.AlterUniqueTogether(
- name='metadata',
- unique_together={('element', 'name', 'index')},
- ),
- migrations.AlterUniqueTogether(
- name='entityrole',
- unique_together={('parent_name', 'child_name', 'parent_type', 'child_type', 'corpus')},
- ),
- migrations.AddConstraint(
- model_name='elementtype',
- constraint=models.UniqueConstraint(fields=('corpus', 'slug'), name='corpus_unique_type_slug'),
- ),
- migrations.AddIndex(
- model_name='elementpath',
- index=django.contrib.postgres.indexes.GinIndex(fields=['path'], name='documents_e_path_15a4b8_gin'),
- ),
- migrations.AlterUniqueTogether(
- name='classification',
- unique_together={('element', 'source', 'ml_class')},
- ),
- migrations.AlterUniqueTogether(
- name='allowedmetadata',
- unique_together={('corpus', 'type', 'name')},
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='selections', to='documents.element'),
),
]
diff --git a/arkindex/documents/migrations/0003_auto_20200218_1118.py b/arkindex/documents/migrations/0003_auto_20200218_1118.py
deleted file mode 100644
index a3396c0aa07c05d61d820eff3da7f55abe9c5876..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0003_auto_20200218_1118.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.2.9 on 2020-02-18 11:18
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0002_initial'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='mlclass',
- name='name',
- field=models.CharField(max_length=1024),
- ),
- ]
diff --git a/arkindex/documents/migrations/0003_initial.py b/arkindex/documents/migrations/0003_initial.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f37c575b7cbb7b51e88572a756ae6ebff23dc9d
--- /dev/null
+++ b/arkindex/documents/migrations/0003_initial.py
@@ -0,0 +1,336 @@
+# Generated by Django 4.1.7 on 2023-05-29 11:49
+
+import django.contrib.gis.db.models.functions
+import django.contrib.postgres.constraints
+import django.contrib.postgres.indexes
+import django.db.models.constraints
+import django.db.models.deletion
+import django.db.models.functions.comparison
+from django.conf import settings
+from django.contrib.postgres.operations import BtreeGistExtension
+from django.db import migrations, models
+
+import arkindex.documents.models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('process', '0002_initial'),
+ ('documents', '0002_initial'),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('images', '0001_initial'),
+ ]
+
+ operations = [
+ BtreeGistExtension(),
+ migrations.AddField(
+ model_name='selection',
+ name='user',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='selections', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='mlclass',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ml_classes', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='metadata',
+ name='element',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metadatas', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='metadata',
+ name='entity',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='metadatas', to='documents.entity'),
+ ),
+ migrations.AddField(
+ model_name='metadata',
+ name='worker_run',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='metadatas', to='process.workerrun'),
+ ),
+ migrations.AddField(
+ model_name='metadata',
+ name='worker_version',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='metadatas', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='entitytype',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entity_types', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='entityrole',
+ name='child_type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='child_role', to='documents.entitytype'),
+ ),
+ migrations.AddField(
+ model_name='entityrole',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='roles', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='entityrole',
+ name='parent_type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='parent_role', to='documents.entitytype'),
+ ),
+ migrations.AddField(
+ model_name='entitylink',
+ name='child',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='children', to='documents.entity'),
+ ),
+ migrations.AddField(
+ model_name='entitylink',
+ name='parent',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='parents', to='documents.entity'),
+ ),
+ migrations.AddField(
+ model_name='entitylink',
+ name='role',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='documents.entityrole'),
+ ),
+ migrations.AddField(
+ model_name='entity',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='entity',
+ name='moderator',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='entities', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='entity',
+ name='type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='entities', to='documents.entitytype'),
+ ),
+ migrations.AddField(
+ model_name='entity',
+ name='worker_run',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='entities', to='process.workerrun'),
+ ),
+ migrations.AddField(
+ model_name='entity',
+ name='worker_version',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='elementtype',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='elementpath',
+ name='element',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='paths', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='element',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='elements', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='element',
+ name='creator',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='elements', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='element',
+ name='image',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='elements', to='images.image'),
+ ),
+ migrations.AddField(
+ model_name='element',
+ name='type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='elements', to='documents.elementtype'),
+ ),
+ migrations.AddField(
+ model_name='element',
+ name='worker_run',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='elements', to='process.workerrun'),
+ ),
+ migrations.AddField(
+ model_name='element',
+ name='worker_version',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='elements', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='corpusexport',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='exports', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='corpusexport',
+ name='user',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='exports', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='corpus',
+ name='top_level_type',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='top_level_corpora', to='documents.elementtype'),
+ ),
+ migrations.AddField(
+ model_name='classification',
+ name='element',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='classifications', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='classification',
+ name='ml_class',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='classifications', to='documents.mlclass'),
+ ),
+ migrations.AddField(
+ model_name='classification',
+ name='moderator',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='classifications', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='classification',
+ name='worker_run',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='classifications', to='process.workerrun'),
+ ),
+ migrations.AddField(
+ model_name='classification',
+ name='worker_version',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='classifications', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='allowedmetadata',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='allowed_metadatas', to='documents.corpus'),
+ ),
+ migrations.AddConstraint(
+ model_name='transcriptionentity',
+ constraint=models.CheckConstraint(check=models.Q(('worker_version_id__isnull', False), ('worker_run_id__isnull', True), _connector='OR'), name='transcription_entity_worker_run_requires_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='transcriptionentity',
+ constraint=models.UniqueConstraint(condition=models.Q(('worker_version_id__isnull', True)), fields=('transcription', 'entity', 'offset', 'length'), name='transcription_entity_unique_manual'),
+ ),
+ migrations.AddConstraint(
+ model_name='transcriptionentity',
+ constraint=models.UniqueConstraint(condition=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False)), fields=('transcription', 'entity', 'offset', 'length', 'worker_version'), name='transcription_entity_unique_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='transcriptionentity',
+ constraint=models.UniqueConstraint(condition=models.Q(('worker_run_id__isnull', False)), fields=('transcription', 'entity', 'offset', 'length', 'worker_run'), name='transcription_entity_unique_worker_run'),
+ ),
+ migrations.AddConstraint(
+ model_name='transcription',
+ constraint=models.CheckConstraint(check=models.Q(('worker_version_id__isnull', False), ('worker_run_id__isnull', True), _connector='OR'), name='transcription_worker_run_requires_worker_version'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='selection',
+ unique_together={('element', 'user')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='mlclass',
+ unique_together={('name', 'corpus')},
+ ),
+ migrations.AddIndex(
+ model_name='metadata',
+ index=models.Index(django.db.models.functions.comparison.Cast('value', output_field=models.FloatField()), condition=models.Q(('type', arkindex.documents.models.MetaType['Numeric'])), name='metadata_numeric_index'),
+ ),
+ migrations.AddConstraint(
+ model_name='metadata',
+ constraint=models.CheckConstraint(check=models.Q(models.Q(('type', arkindex.documents.models.MetaType['Numeric']), _negated=True), ('value__iregex', '^[+-]?(?:\\d+(?:\\.\\d*)?|\\.\\d+)(?:E[+-]?\\d+)?$'), _connector='OR'), name='metadata_numeric_values'),
+ ),
+ migrations.AddConstraint(
+ model_name='metadata',
+ constraint=models.CheckConstraint(check=models.Q(('worker_version_id__isnull', False), ('worker_run_id__isnull', True), _connector='OR'), name='metadata_worker_run_requires_worker_version'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='entitytype',
+ unique_together={('name', 'corpus')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='entityrole',
+ unique_together={('parent_name', 'child_name', 'parent_type', 'child_type', 'corpus')},
+ ),
+ migrations.AddIndex(
+ model_name='entity',
+ index=models.Index(fields=['corpus_id', 'name', 'id'], name='entity_list_index'),
+ ),
+ migrations.AddConstraint(
+ model_name='entity',
+ constraint=models.CheckConstraint(check=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False), _connector='OR'), name='entity_worker_run_requires_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='elementtype',
+ constraint=models.UniqueConstraint(fields=('corpus', 'slug'), name='corpus_unique_type_slug'),
+ ),
+ migrations.AddIndex(
+ model_name='elementpath',
+ index=django.contrib.postgres.indexes.GinIndex(fields=['path'], name='documents_e_path_15a4b8_gin'),
+ ),
+ migrations.AddIndex(
+ model_name='elementpath',
+ index=models.Index(models.F('path__last'), name='element_path_last'),
+ ),
+ migrations.AddConstraint(
+ model_name='elementpath',
+ constraint=models.UniqueConstraint(fields=('element', 'path'), name='unique_element_paths'),
+ ),
+ migrations.AddConstraint(
+ model_name='elementpath',
+ constraint=django.contrib.postgres.constraints.ExclusionConstraint(deferrable=django.db.models.constraints.Deferrable['DEFERRED'], expressions=[('element', '='), (django.db.models.functions.comparison.Least('path__len', 1), '<>')], name='unique_top_level'),
+ ),
+ migrations.AddConstraint(
+ model_name='elementpath',
+ constraint=django.contrib.postgres.constraints.ExclusionConstraint(deferrable=django.db.models.constraints.Deferrable['DEFERRED'], expressions=[('element', '='), ('path__last', '='), ('ordering', '<>')], name='unique_element_orderings'),
+ ),
+ migrations.AddConstraint(
+ model_name='elementpath',
+ constraint=django.contrib.postgres.constraints.ExclusionConstraint(deferrable=django.db.models.constraints.Deferrable['DEFERRED'], expressions=[('path__last', '='), ('ordering', '='), ('element', '<>')], name='unique_parent_orderings'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(models.Q(('image_id', None), ('polygon', None)), models.Q(models.Q(('image_id', None), ('polygon', None), _connector='OR'), _negated=True), _connector='OR'), name='element_image_and_polygon'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(('polygon', None), ('polygon__isclosed', True), _connector='OR'), name='element_polygon_closed'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(('polygon', None), ('polygon', django.contrib.gis.db.models.functions.SnapToGrid('polygon', 1)), _connector='OR'), name='element_polygon_integer_coordinates'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(('polygon', None), ('polygon__numpoints__gte', 4), _connector='OR'), name='element_polygon_size'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(('rotation_angle__gte', 0), ('rotation_angle__lte', 359)), name='element_rotation_angle'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(('creator_id', None), ('worker_version_id', None), _connector='OR'), name='element_creator_nand_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='element',
+ constraint=models.CheckConstraint(check=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False), _connector='OR'), name='element_worker_run_requires_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='classification',
+ constraint=models.CheckConstraint(check=models.Q(('worker_version_id__isnull', False), ('worker_run_id__isnull', True), _connector='OR'), name='classification_worker_run_requires_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='classification',
+ constraint=models.UniqueConstraint(condition=models.Q(('worker_version_id__isnull', True)), fields=('element', 'ml_class'), name='classification_unique_manual'),
+ ),
+ migrations.AddConstraint(
+ model_name='classification',
+ constraint=models.UniqueConstraint(condition=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False)), fields=('element', 'ml_class', 'worker_version'), name='classification_unique_worker_version'),
+ ),
+ migrations.AddConstraint(
+ model_name='classification',
+ constraint=models.UniqueConstraint(condition=models.Q(('worker_run_id__isnull', False)), fields=('element', 'ml_class', 'worker_run'), name='classification_unique_worker_run'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='allowedmetadata',
+ unique_together={('corpus', 'type', 'name')},
+ ),
+ ]
diff --git a/arkindex/documents/migrations/0004_remove_elementtype_hidden.py b/arkindex/documents/migrations/0004_remove_elementtype_hidden.py
deleted file mode 100644
index 6307e1346a5f579119b8b710f412ac13060fd9e0..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0004_remove_elementtype_hidden.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 2.2.10 on 2020-03-05 13:20
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0003_auto_20200218_1118'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='elementtype',
- name='hidden',
- ),
- ]
diff --git a/arkindex/documents/migrations/0005_element_source.py b/arkindex/documents/migrations/0005_element_source.py
deleted file mode 100644
index f29467f574da7d6a80a557cadb2cb12bdc917a7b..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0005_element_source.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Generated by Django 2.2.10 on 2020-03-10 10:17
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0004_remove_elementtype_hidden'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='source',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='elements',
- to='documents.DataSource',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0006_preflight.py b/arkindex/documents/migrations/0006_preflight.py
deleted file mode 100644
index 8266824a1927d868403089abdafe28db7c5dc3c8..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0006_preflight.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0005_element_source'),
- ]
-
- operations = [
- # Remove extra regions tied to several elements
- # We can't to keep the first region element item to preserve the link
- # for the followup migration
- migrations.RunSQL(
- """
- SELECT region_id, array_agg(id) as ids into remove FROM documents_regionelement
- GROUP BY region_id
- HAVING COUNT(region_id) > 1;
-
- DELETE FROM documents_regionelement WHERE id in
- (select unnest(ids[2:]) from remove);
-
- DROP table remove;
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0006_region_fk.py b/arkindex/documents/migrations/0006_region_fk.py
deleted file mode 100644
index fdb8f4e27150c72d1031f51b2c28a4208bd7c450..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0006_region_fk.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from django.db import migrations, models
-
-
-def check_m2m(apps, schema_editor):
- """
- Ensure the RegionElement many-to-many relationship only has one
- element per region, making it possible to turn it into a foreign key.
- """
- db_alias = schema_editor.connection.alias
- RegionElement = apps.get_model('documents', 'RegionElement')
- duplicates = RegionElement \
- .objects \
- .using(db_alias) \
- .values('region_id') \
- .annotate(count=models.Count('id')) \
- .filter(count__gt=1)
- assert not duplicates.exists(), 'Some regions are linked to more than one element.'
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0006_preflight'),
- ]
-
- operations = [
- # Initial check before turning the M2M into a foreign key
- migrations.RunPython(
- check_m2m,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- # Rename the Regions M2M to avoid a collision on Element.regions
- migrations.AlterField(
- model_name='Region',
- name='elements',
- field=models.ManyToManyField(
- related_name='regions_m2m',
- through='documents.RegionElement',
- to='documents.Element',
- ),
- ),
- # Add the new foreign key
- migrations.AddField(
- model_name='Region',
- name='element',
- field=models.ForeignKey(
- on_delete=models.deletion.CASCADE,
- related_name='regions',
- to='documents.Element',
- null=True,
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0007_fill_region_fk.py b/arkindex/documents/migrations/0007_fill_region_fk.py
deleted file mode 100644
index 9c5d736b73ff22768c51a2e280a2d87fe253b2d8..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0007_fill_region_fk.py
+++ /dev/null
@@ -1,24 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0006_region_fk'),
- ]
-
- operations = [
- # Move RegionElement.element_id to Region.element_id
- # and remove regions that will not be migrated into elements
- migrations.RunSQL(
- """
- UPDATE documents_region
- SET element_id = documents_regionelement.element_id
- FROM documents_regionelement
- WHERE documents_regionelement.region_id = documents_region.id;
- DELETE FROM documents_region WHERE element_id IS NULL;
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0008_remove_regionelement.py b/arkindex/documents/migrations/0008_remove_regionelement.py
deleted file mode 100644
index ffa89a28b9175e07e60ea8f2684e0b957b5bd281..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0008_remove_regionelement.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0007_fill_region_fk'),
- ]
-
- operations = [
- # Make the foreign key non-nullable
- migrations.AlterField(
- model_name='Region',
- name='element',
- field=models.ForeignKey(
- on_delete=models.deletion.CASCADE,
- related_name='regions',
- to='documents.Element',
- ),
- ),
- # Drop RegionElement
- migrations.RemoveField(
- model_name='Region',
- name='elements',
- ),
- migrations.AlterUniqueTogether(
- name='RegionElement',
- unique_together=None,
- ),
- migrations.RemoveField(
- model_name='RegionElement',
- name='element',
- ),
- migrations.RemoveField(
- model_name='RegionElement',
- name='region',
- ),
- migrations.DeleteModel(
- name='RegionElement',
- ),
- ]
diff --git a/arkindex/documents/migrations/0009_migrate_regions.py b/arkindex/documents/migrations/0009_migrate_regions.py
deleted file mode 100644
index 92942b0f42e7cadd30e0b30737f7ce236934771b..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0009_migrate_regions.py
+++ /dev/null
@@ -1,84 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0008_remove_regionelement'),
- ]
-
- operations = [
- # This performs the whole regions migration in just a few SQL queries
- # by exploiting the fact that UUIDs have a lot of combinations and it is
- # reasonable to assume that they are unique even across tables.
-
- # This turns MLClass IDs into element type IDs and region IDs into element IDs.
- # This way, no checks are required to ensure the foreign key constraints are respected.
- # The element names are generated as the class name and a number grouped by parent element.
- # Then, element paths are created in two steps:
- # * Create the paths from scratch for regions whose elements do not have paths;
- # * Create the paths for regions whose elements already have paths by prepending the element ID to all paths.
- # We do not have to care about existing paths, cycles or existing orderings since
- # everything is in a transaction and uses new types.
-
- # The uuid-ossp Postgres extension is required to generate new UUIDs
- # for classifications as those are created from scratch
- # and Django handles defaults only in Python, not in the database.
-
- migrations.RunSQL(
- """
- CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-
- INSERT INTO documents_elementtype (id, corpus_id, slug, display_name, folder, default_view)
- SELECT id, corpus_id, 'region_' || name, name || ' region', FALSE, FALSE
- FROM documents_mlclass
- WHERE id in (SELECT DISTINCT ml_class_id FROM documents_region);
-
- INSERT INTO documents_element (id, corpus_id, type_id, name, zone_id, source_id, created, updated)
- SELECT
- region.id,
- ml_class.corpus_id,
- region.ml_class_id,
- ml_class.name || ' ' || ROW_NUMBER() OVER (
- PARTITION BY region.element_id, region.ml_class_id ORDER BY region.id
- ),
- region.zone_id,
- region.source_id,
- CURRENT_TIMESTAMP,
- CURRENT_TIMESTAMP
- FROM documents_region region
- LEFT JOIN documents_mlclass ml_class
- ON (region.ml_class_id = ml_class.id);
-
- INSERT INTO documents_classification
- (id, element_id, source_id, ml_class_id, confidence, high_confidence, state)
- SELECT uuid_generate_v4(), id, source_id, ml_class_id, confidence, TRUE, 'pending'
- FROM documents_region;
-
- INSERT INTO documents_elementpath (id, element_id, path, ordering)
- SELECT
- uuid_generate_v4(),
- id,
- ARRAY[element_id],
- ROW_NUMBER() OVER (
- PARTITION BY element_id, ml_class_id ORDER BY id
- ) - 1
- FROM documents_region
- WHERE element_id NOT IN (SELECT element_id FROM documents_elementpath);
-
- INSERT INTO documents_elementpath (id, element_id, path, ordering)
- SELECT
- uuid_generate_v4(),
- region.id,
- ep.path || element_id,
- ROW_NUMBER() OVER (
- PARTITION BY element_id, region.ml_class_id, ep.path[array_length(ep.path, 1)] ORDER BY region.id
- ) - 1
- FROM documents_elementpath ep
- FULL JOIN documents_region region USING (element_id)
- WHERE region.id IS NOT NULL;
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0010_remove_region.py b/arkindex/documents/migrations/0010_remove_region.py
deleted file mode 100644
index 44840b792bbd83040aa6e767226a9c21053c5ccb..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0010_remove_region.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Generated by Django 2.2.10 on 2020-03-18 10:09
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0009_migrate_regions'),
- ]
-
- operations = [
- migrations.DeleteModel(
- name='Region',
- ),
- ]
diff --git a/arkindex/documents/migrations/0011_auto_20200403_1456.py b/arkindex/documents/migrations/0011_auto_20200403_1456.py
deleted file mode 100644
index de7be88dc8759b64e9a1b714fc81c14c47fd6481..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0011_auto_20200403_1456.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Generated by Django 2.2.10 on 2020-04-03 14:56
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0010_remove_region'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='transcription',
- name='text',
- field=models.TextField(),
- ),
- migrations.AlterField(
- model_name='transcription',
- name='zone',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.PROTECT,
- related_name='transcriptions',
- to='images.Zone',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0012_add_created_updated_corpus.py b/arkindex/documents/migrations/0012_add_created_updated_corpus.py
deleted file mode 100644
index 45ba4088ab9b6e03fbf81a127b2e3b75f31131d5..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0012_add_created_updated_corpus.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Generated by Django 2.2.11 on 2020-05-08 07:51
-
-import django.utils.timezone
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0011_auto_20200403_1456'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='corpus',
- name='created',
- field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
- preserve_default=False,
- ),
- migrations.AddField(
- model_name='corpus',
- name='updated',
- field=models.DateTimeField(auto_now=True),
- ),
- ]
diff --git a/arkindex/documents/migrations/0013_datasource_type_length.py b/arkindex/documents/migrations/0013_datasource_type_length.py
deleted file mode 100644
index d9b10a58b7003c7dc119244b83a1503ae9077bad..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0013_datasource_type_length.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 2.2.10 on 2020-05-19 14:59
-
-from enum import Enum
-
-import enumfields.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0012_add_created_updated_corpus'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='datasource',
- name='type',
- field=enumfields.fields.EnumField(enum=Enum('MLToolType', ''), max_length=50),
- ),
- ]
diff --git a/arkindex/documents/migrations/0014_nullable_transcription_zone.py b/arkindex/documents/migrations/0014_nullable_transcription_zone.py
deleted file mode 100644
index b85d8fd846fbdfe76eb052af3cb1c5800d784430..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0014_nullable_transcription_zone.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 2.2.9 on 2020-05-27 09:03
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0013_datasource_type_length'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='transcription',
- name='zone',
- field=models.ForeignKey(
- null=True,
- on_delete=django.db.models.deletion.PROTECT,
- related_name='transcriptions',
- to='images.Zone'
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0015_elementtype_allowed_transcription.py b/arkindex/documents/migrations/0015_elementtype_allowed_transcription.py
deleted file mode 100644
index cec7e56a12fe7c5b4cf9a7d0dfbf5e7f63153c0e..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0015_elementtype_allowed_transcription.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Generated by Django 2.2.9 on 2020-06-03 13:26
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0014_nullable_transcription_zone'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='elementtype',
- name='allowed_transcription',
- field=models.CharField(
- blank=True,
- max_length=10,
- null=True
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0016_selection.py b/arkindex/documents/migrations/0016_selection.py
deleted file mode 100644
index 8f90a4ef6a37de4ce75c7c3d43921b2222f85f64..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0016_selection.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-23 14:07
-
-import uuid
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('documents', '0015_elementtype_allowed_transcription'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Selection',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('element', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='selections',
- to='documents.Element',
- )),
- ('user', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='selections',
- to=settings.AUTH_USER_MODEL,
- )),
- ],
- options={
- 'unique_together': {('element', 'user')},
- },
- ),
- ]
diff --git a/arkindex/documents/migrations/0017_remove_elementtype_default_view.py b/arkindex/documents/migrations/0017_remove_elementtype_default_view.py
deleted file mode 100644
index 96154183554fc8437e47842c9fcaaeab5319b7c1..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0017_remove_elementtype_default_view.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-25 12:39
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0016_selection'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='elementtype',
- name='default_view',
- ),
- ]
diff --git a/arkindex/documents/migrations/0018_worker_version_attributes.py b/arkindex/documents/migrations/0018_worker_version_attributes.py
deleted file mode 100644
index c0d670af4a529d11af6d119236c01ceb4a9bd6ab..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0018_worker_version_attributes.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Generated by Django 2.2.13 on 2020-08-10 10:00
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0015_clear_payload'),
- ('documents', '0017_remove_elementtype_default_view'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='worker_version',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='elements', to='process.WorkerVersion'),
- ),
- migrations.AddField(
- model_name='transcription',
- name='worker_version',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='process.WorkerVersion'),
- ),
- migrations.AlterField(
- model_name='transcription',
- name='source',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='documents.DataSource'),
- ),
- migrations.AddField(
- model_name='classification',
- name='worker_version',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='classifications', to='process.WorkerVersion'),
- ),
- migrations.AlterField(
- model_name='classification',
- name='source',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='classifications', to='documents.DataSource'),
- ),
- migrations.AlterUniqueTogether(
- name='classification',
- unique_together=set(),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(condition=models.Q(worker_version_id__isnull=True), fields=('element', 'ml_class', 'source'), name='classification_unique_source'),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(condition=models.Q(source_id__isnull=True), fields=('element', 'ml_class', 'worker_version'), name='classification_unique_worker_version'),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.CheckConstraint(check=models.Q(models.Q(('source_id__isnull', False), ('worker_version_id__isnull', True)), models.Q(('source_id__isnull', True), ('worker_version_id__isnull', False)), _connector='OR'), name='classification_source_xor_workerversion'),
- ),
- migrations.AddField(
- model_name='entity',
- name='worker_version',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='process.WorkerVersion'),
- ),
- migrations.AlterField(
- model_name='entity',
- name='source',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='documents.DataSource'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0019_corpus_repository.py b/arkindex/documents/migrations/0019_corpus_repository.py
deleted file mode 100644
index 350c4465f4d7ed4fb4e59fa037d1421b7eff497f..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0019_corpus_repository.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Generated by Django 3.1 on 2020-09-11 07:26
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-import arkindex.process.models
-
-
-def copy_repository_corpus(apps, schema_editor):
- Corpus = apps.get_model('documents', 'Corpus')
- for corpus in Corpus.objects.all():
- repos = list(corpus.repos.filter(type=arkindex.process.models.RepositoryType.IIIF))
- if len(repos) == 0:
- continue
- elif len(repos) == 1:
- corpus.repository = repos[0]
- else:
- raise Exception(f'"{corpus.name}" corpus is referenced by multiple IIIF repositories: {[r.url for r in repos]}')
- corpus.save()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0019_repository_type'),
- ('documents', '0018_worker_version_attributes'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='corpus',
- name='repository',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='corpora', to='process.repository'),
- ),
- migrations.RunPython(
- copy_repository_corpus,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/documents/migrations/0020_remove_source_xor_version_constraint.py b/arkindex/documents/migrations/0020_remove_source_xor_version_constraint.py
deleted file mode 100644
index 8907ff7578fcfdd2237963ec17610a08c7470a2c..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0020_remove_source_xor_version_constraint.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by Django 3.1 on 2020-10-14 13:56
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0019_corpus_repository'),
- ]
-
- operations = [
- migrations.RemoveConstraint(
- model_name='classification',
- name='classification_unique_source',
- ),
- migrations.RemoveConstraint(
- model_name='classification',
- name='classification_unique_worker_version',
- ),
- migrations.RemoveConstraint(
- model_name='classification',
- name='classification_source_xor_workerversion',
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(condition=models.Q(('source_id__isnull', True), ('worker_version_id__isnull', True)), fields=('element', 'ml_class'), name='classification_unique_manual'),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(condition=models.Q(('source_id__isnull', True), ('worker_version_id__isnull', False)), fields=('element', 'ml_class', 'worker_version'), name='classification_unique_worker_version'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0021_move_transcriptions.py b/arkindex/documents/migrations/0021_move_transcriptions.py
deleted file mode 100644
index 7c4566b39314e7640952d0951298abd2b34fa255..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0021_move_transcriptions.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Generated by Django 3.1 on 2020-09-01 07:48
-
-from django.db import migrations, models
-
-
-def preflight_checks(apps, schema_editor):
- ElementType = apps.get_model('documents', 'ElementType')
- Transcription = apps.get_model('documents', 'Transcription')
- existing_types = []
-
- for ts_type in Transcription.objects.values('type').distinct():
- if ElementType.objects.filter(slug=f'transcription_{ts_type}').exists():
- existing_types.append(f'`transcription_{ts_type}`')
-
- if existing_types:
- raise AssertionError(
- 'This migration could not be run because one or more element types use the reserved slug(s) '
- + ', '.join(existing_types)
- )
-
-
-FORWARD_SQL = [
- 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";',
- # Early handling for the edge case of transcriptions already on the correct element
- """
- UPDATE documents_transcription transcription
- SET zone_id = NULL
- FROM documents_element element
- WHERE transcription.element_id = element.id
- AND transcription.zone_id IS NOT NULL
- AND transcription.zone_id = element.zone_id;
- """,
- # Create element types starting with transcription_* as needed
- """
- INSERT INTO documents_elementtype (id, corpus_id, slug, display_name, folder, allowed_transcription)
- SELECT
- uuid_generate_v4(),
- element.corpus_id,
- 'transcription_' || transcription.type,
- initcap(transcription.type) || ' Transcription',
- FALSE,
- transcription.type
- FROM documents_transcription transcription
- INNER JOIN documents_element element ON (element.id = transcription.element_id)
- WHERE transcription.zone_id IS NOT NULL
- GROUP BY element.corpus_id, transcription.type;
- """,
- # Create new elements
- """
- INSERT INTO documents_element (id, corpus_id, type_id, name, zone_id, source_id, worker_version_id, created, updated)
- SELECT
- transcription.id,
- element.corpus_id,
- type.id,
- (ROW_NUMBER() OVER (
- PARTITION BY
- transcription.element_id,
- transcription.source_id,
- transcription.worker_version_id,
- transcription.type
- ORDER BY
- ST_Y(ST_StartPoint(polygon)),
- ST_X(ST_StartPoint(polygon))
- ))::varchar,
- transcription.zone_id,
- transcription.source_id,
- transcription.worker_version_id,
- NOW(),
- NOW()
- FROM
- documents_transcription transcription
- INNER JOIN documents_element element on (transcription.element_id = element.id)
- INNER JOIN documents_elementtype type ON (type.corpus_id = element.corpus_id AND type.slug = 'transcription_' || transcription.type)
- INNER JOIN images_zone zone ON (transcription.zone_id = zone.id);
- """,
- # Create element paths
- # Append to existing parent paths of the parent element, or create one new element path with the parent element itself in it
- """
- INSERT INTO documents_elementpath (id, element_id, path, ordering)
- SELECT
- uuid_generate_v4(),
- transcription.id,
- COALESCE(path.path, ARRAY[]::uuid[]) || transcription.element_id,
- ROW_NUMBER() OVER (
- PARTITION BY transcription.element_id
- ORDER BY
- ST_Y(ST_StartPoint(polygon)),
- ST_X(ST_StartPoint(polygon))
- )
- FROM
- documents_transcription transcription
- INNER JOIN images_zone zone ON (zone.id = transcription.zone_id)
- LEFT JOIN documents_elementpath path ON (path.element_id = transcription.element_id);
- """,
- # Move transcriptions to their new elements
- """
- UPDATE documents_transcription
- SET element_id = id
- WHERE zone_id IS NOT NULL;
- """,
- # At this point, we can drop the zone column, but this would fail due to 'pending trigger events'
- # Postgres does not allow editing the schema *after* editing the data in the same transcription;
- # This migration is continued in documents.0021 to allow a new database transaction to happen.
-]
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0020_remove_source_xor_version_constraint'),
- ('images', '0005_polygon_index')
- ]
-
- operations = [
- migrations.AddConstraint(
- model_name='transcription',
- constraint=models.CheckConstraint(
- check=~models.Q(source_id__isnull=False, worker_version_id__isnull=False),
- name='transcription_source_not_worker_version',
- )
- ),
- migrations.RunPython(
- preflight_checks,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.RunSQL(
- FORWARD_SQL,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0022_remove_transcription_zone.py b/arkindex/documents/migrations/0022_remove_transcription_zone.py
deleted file mode 100644
index edb333dab4f65cef1068ba654291e1468a32a201..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0022_remove_transcription_zone.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# Generated by Django 3.1 on 2020-09-01 07:48
-
-from django.db import migrations
-
-FORWARD_SQL = [
- # Use a temporary table here to iterate over transcriptions just once before deleting,
- # causing this migration to only take a few minutes
- # Note the strange join conditions as either source_id or worker_version_id are NULL,
- # which causes a NATURAL JOIN or a JOIN … USING to fail since comparing NULLs returns NULL.
- """
- CREATE TEMPORARY TABLE duplicate_ids AS
- WITH filters AS (
- SELECT
- sub.*,
- FIRST_VALUE(id) OVER (
- PARTITION BY
- transcription.element_id,
- transcription.source_id,
- transcription.worker_version_id
- ) AS keep_id
- FROM documents_transcription transcription
- INNER JOIN (
- SELECT element_id, source_id, worker_version_id
- FROM documents_transcription
- GROUP BY element_id, source_id, worker_version_id
- HAVING COUNT(*) > 1
- ) sub ON (
- sub.element_id = transcription.element_id AND (
- sub.source_id = transcription.source_id
- OR sub.worker_version_id = transcription.worker_version_id
- )
- )
- )
- SELECT id
- FROM documents_transcription transcription
- INNER JOIN filters ON (
- filters.element_id = transcription.element_id AND (
- filters.source_id = transcription.source_id
- OR filters.worker_version_id = transcription.worker_version_id
- )
- )
- WHERE keep_id != id;
- """,
- # Remove any TranscriptionEntity that could be linked to the duplicate transcriptions
- """
- DELETE FROM documents_transcriptionentity transcriptionentity
- USING duplicate_ids
- WHERE transcriptionentity.transcription_id = duplicate_ids.id;
- """,
- # Remove duplicate transcriptions
- """
- DELETE FROM documents_transcription transcription
- USING duplicate_ids
- WHERE transcription.id = duplicate_ids.id;
- """,
- 'DROP TABLE duplicate_ids;',
-]
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0021_move_transcriptions'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='transcription',
- name='zone',
- ),
- # Remove the few remaining transcriptions that would break the unique constraints we will add in documents.0023.
- # Those are transcriptions from the same source, on the same element, with the exact same zones.
- # This query is rather complex as we want to only remove duplicates, and window functions have their limits,
- # but the GROUP BY…HAVING will quickly exclude most of the table so it isn't slow.
- migrations.RunSQL(
- FORWARD_SQL,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0023_remove_transcription_type.py b/arkindex/documents/migrations/0023_remove_transcription_type.py
deleted file mode 100644
index 7d1c474b016cf59e1cf22da9a70ac1c7b55166ae..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0023_remove_transcription_type.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 3.1.3 on 2020-11-23 14:35
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0022_remove_transcription_zone'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='elementtype',
- name='allowed_transcription',
- ),
- migrations.RemoveField(
- model_name='transcription',
- name='type',
- ),
- ]
diff --git a/arkindex/documents/migrations/0024_migrate_datasource.py b/arkindex/documents/migrations/0024_migrate_datasource.py
deleted file mode 100644
index f52bb4c9cfeac673806275e1570906c237783686..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0024_migrate_datasource.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# Generated by Django 3.1.3 on 2020-11-30 10:40
-
-import os
-import uuid
-
-from django.db import migrations, models
-
-
-def migrate_data_sources(apps, schema_editor):
- DataSource = apps.get_model('documents', 'DataSource')
- if not DataSource.objects.exists():
- return
-
- repo_prefix = os.environ.get('REPOSITORY_PREFIX', 'https://gitlab.com/teklia/workers')
-
- Element = apps.get_model('documents', 'Element')
- Classification = apps.get_model('documents', 'Classification')
- Transcription = apps.get_model('documents', 'Transcription')
- Entity = apps.get_model('documents', 'Entity')
- Repository = apps.get_model('process', 'Repository')
- Worker = apps.get_model('process', 'Worker')
-
- # Start by just removing the manual source, since a manual worker version is just None
- print('Migrating manual sources…')
- Element.objects.filter(source__slug='manual').update(source=None)
- Classification.objects.filter(source__slug='manual').update(source=None)
- Transcription.objects.filter(source__slug='manual').update(source=None)
- Entity.objects.filter(source__slug='manual').update(source=None)
-
- # Only migrate sources that have related objects
- to_migrate = DataSource.objects.filter(
- id__in=Element.objects.values('source_id').union(
- Classification.objects.values('source_id')
- ).union(
- Transcription.objects.values('source_id')
- ).union(
- Entity.objects.values('source_id')
- )
- )
- for source in to_migrate:
- print(f'Migrating {source.name} {source.revision} ({source.id})…')
- repo, _ = Repository.objects.get_or_create(
- url=f'{repo_prefix}/{source.slug}',
- type='worker',
- defaults={
- # This is supposed to be unique
- 'hook_token': str(source.id),
- },
- )
- worker, _ = Worker.objects.get_or_create(
- repository=repo,
- slug=source.slug,
- defaults={
- 'name': source.name,
- 'type': str(source.type),
- }
- )
-
- # To ensure we cannot get duplicate worker version IDs when re-applying
- # unique constraints later, we just keep on trying to get a new revision.
- created = False
- while not created:
- revision, created = repo.revisions.get_or_create(
- hash=uuid.uuid4().hex,
- message='Migrated DataSource',
- author='Arkindex',
- )
-
- version = worker.workerversion_set.create(
- revision=revision,
- configuration={},
- )
-
- source.elements.update(source=None, worker_version=version)
- source.classifications.update(source=None, worker_version=version)
- source.transcriptions.update(source=None, worker_version=version)
- source.entities.update(source=None, worker_version=version)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0023_remove_transcription_type'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='DataSource',
- name='type',
- field=models.CharField(max_length=50),
- ),
- migrations.RemoveConstraint(
- model_name='classification',
- name='classification_unique_manual',
- ),
- migrations.RemoveConstraint(
- model_name='classification',
- name='classification_unique_worker_version',
- ),
- migrations.RemoveConstraint(
- model_name='transcription',
- name='transcription_source_not_worker_version',
- ),
- migrations.RunPython(
- code=migrate_data_sources,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- # Duplicate classifications may still remain
- migrations.RunSQL(
- """
- DELETE FROM documents_classification WHERE id IN (
- SELECT id
- FROM (
- SELECT c.id, ROW_NUMBER() OVER (
- PARTITION BY x.element_id
- ORDER BY c.confidence DESC
- ) AS nb
- FROM (
- SELECT element_id, ml_class_id
- FROM documents_classification
- GROUP BY element_id, ml_class_id
- HAVING COUNT(id) > 1
- ) AS x
- INNER JOIN documents_classification AS c USING (element_id, ml_class_id)
- ) AS y
- WHERE y.nb > 1
- );
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- # Deletion happens in another migration, since updating data then trying to update
- # the structure causes errors with 'pending trigger events'
- ]
diff --git a/arkindex/documents/migrations/0025_drop_datasource.py b/arkindex/documents/migrations/0025_drop_datasource.py
deleted file mode 100644
index 2f0bd0c09011bb065b0928b4fb156b33a8e98f20..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0025_drop_datasource.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Generated by Django 3.1.3 on 2020-12-10 13:57
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0024_migrate_datasource'),
- ]
-
- operations = [
- migrations.AlterUniqueTogether(
- name='datasource',
- unique_together=None,
- ),
- migrations.RemoveField(
- model_name='classification',
- name='source',
- ),
- migrations.RemoveField(
- model_name='element',
- name='source',
- ),
- migrations.RemoveField(
- model_name='entity',
- name='source',
- ),
- migrations.RemoveField(
- model_name='transcription',
- name='source',
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(
- condition=models.Q(worker_version_id__isnull=True),
- fields=('element', 'ml_class'),
- name='classification_unique_manual',
- ),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(
- condition=models.Q(worker_version_id__isnull=False),
- fields=('element', 'ml_class', 'worker_version'),
- name='classification_unique_worker_version',
- ),
- ),
- migrations.DeleteModel(
- name='DataSource',
- ),
- ]
diff --git a/arkindex/documents/migrations/0026_metadata_worker_version.py b/arkindex/documents/migrations/0026_metadata_worker_version.py
deleted file mode 100644
index f419f9f370d92257dcf3e17c7e55819779ea13b3..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0026_metadata_worker_version.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 3.1.4 on 2020-12-30 15:31
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0025_dataimport_name'),
- ('documents', '0025_drop_datasource'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='metadata',
- name='worker_version',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='metadatas', to='process.workerversion'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0027_remove_metadata_revision.py b/arkindex/documents/migrations/0027_remove_metadata_revision.py
deleted file mode 100644
index 204826e1a697406d2c17acb44ac95be35a0cc82b..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0027_remove_metadata_revision.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 3.1.5 on 2021-01-29 09:23
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0026_metadata_worker_version'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='metadata',
- name='revision',
- ),
- ]
diff --git a/arkindex/documents/migrations/0028_transcription_confidence.py b/arkindex/documents/migrations/0028_transcription_confidence.py
deleted file mode 100644
index 54114c8f41f7e3f2dd9d1d772f05e3c46d36accb..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0028_transcription_confidence.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.6 on 2021-02-15 15:34
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0027_remove_metadata_revision'),
- ]
-
- operations = [
- migrations.RenameField(
- model_name='transcription',
- old_name='score',
- new_name='confidence',
- ),
- ]
diff --git a/arkindex/documents/migrations/0029_corpus_top_level_type.py b/arkindex/documents/migrations/0029_corpus_top_level_type.py
deleted file mode 100644
index cd2dd87d7a5f5bc133bf5e436a5ce4375e4d166c..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0029_corpus_top_level_type.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.1.5 on 2021-03-04 08:45
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0028_transcription_confidence'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='corpus',
- name='top_level_type',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='top_level_corpora', to='documents.elementtype'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0030_convert_html_metadata_as_markdown.py b/arkindex/documents/migrations/0030_convert_html_metadata_as_markdown.py
deleted file mode 100644
index 87d49a4fe7012c3d88405517f96563792bea533a..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0030_convert_html_metadata_as_markdown.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by Django 3.1.7 on 2021-03-17 08:26
-from django.db import migrations
-
-from arkindex.documents.models import MetaType
-
-
-def convert_html_metadata_to_markdown(apps, schema_editor):
- MetaData = apps.get_model('documents', 'MetaData')
- AllowedMetaData = apps.get_model('documents', 'AllowedMetaData')
- MetaData.objects.exclude(type__in=[mt.value for mt in MetaType]).update(type=MetaType.Markdown)
- AllowedMetaData.objects.exclude(type__in=[mt.value for mt in MetaType]).update(type=MetaType.Markdown)
-
-
-def convert_markdown_metadata_to_html(apps, schema_editor):
- MetaData = apps.get_model('documents', 'MetaData')
- AllowedMetaData = apps.get_model('documents', 'AllowedMetaData')
- MetaData.objects.exclude(type__in=[mt.value for mt in MetaType]).update(type=MetaType.HTML)
- AllowedMetaData.objects.exclude(type__in=[mt.value for mt in MetaType]).update(type=MetaType.HTML)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0029_corpus_top_level_type'),
- ]
-
- operations = [
- migrations.RunPython(
- convert_html_metadata_to_markdown,
- reverse_code=convert_markdown_metadata_to_html,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0031_add_indexable_fields.py b/arkindex/documents/migrations/0031_add_indexable_fields.py
deleted file mode 100644
index daee5457e4dce1d2e0fc34fc5fd42de7a015ea9b..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0031_add_indexable_fields.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 3.1.7 on 2021-04-08 09:01
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0030_convert_html_metadata_as_markdown'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='corpus',
- name='indexable',
- field=models.BooleanField(default=False),
- ),
- migrations.AddField(
- model_name='elementtype',
- name='indexable',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/documents/migrations/0032_alter_transcriptionentity_length.py b/arkindex/documents/migrations/0032_alter_transcriptionentity_length.py
deleted file mode 100644
index 955d7d65e98015a425026db6fcaf1abc202525ec..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0032_alter_transcriptionentity_length.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.1.7 on 2021-04-28 10:10
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0031_add_indexable_fields'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='transcriptionentity',
- name='length',
- field=models.PositiveIntegerField(validators=[django.core.validators.MinValueValidator(1)]),
- ),
- ]
diff --git a/arkindex/documents/migrations/0033_corpus_thumbnail.py b/arkindex/documents/migrations/0033_corpus_thumbnail.py
deleted file mode 100644
index 5fe75cd292b3bff9dccd04746cbc1d727d00edca..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0033_corpus_thumbnail.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 3.1.8 on 2021-04-16 13:29
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0032_alter_transcriptionentity_length'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='corpus',
- name='thumbnail',
- field=models.OneToOneField(
- blank=True,
- help_text='Optional element used as the thumbnail for this corpus',
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='thumbnail_corpus',
- to='documents.element',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0034_transcriptionentity_worker_version.py b/arkindex/documents/migrations/0034_transcriptionentity_worker_version.py
deleted file mode 100644
index be4222ae3ecc3d074f0299d3db17080ed429906e..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0034_transcriptionentity_worker_version.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Generated by Django 3.1.8 on 2021-05-05 15:27
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0032_dataimport_activity_state'),
- ('documents', '0033_corpus_thumbnail'),
- ]
-
- atomic = False
-
- operations = [
- migrations.AddField(
- model_name='transcriptionentity',
- name='worker_version',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name='transcription_entities',
- to='process.workerversion'
- ),
- ),
- migrations.AlterUniqueTogether(
- name='transcriptionentity',
- unique_together={
- ('transcription', 'entity', 'offset', 'length', 'worker_version')
- },
- ),
- migrations.RunSQL(
- """
- UPDATE documents_transcriptionentity tr_entity
- SET worker_version_id = entity.worker_version_id
- FROM documents_entity entity
- WHERE tr_entity.entity_id = entity.id;
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- )
- ]
diff --git a/arkindex/documents/migrations/0035_corpusexport.py b/arkindex/documents/migrations/0035_corpusexport.py
deleted file mode 100644
index 2af867c18621f52351039975528ba39acc21f7b4..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0035_corpusexport.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# Generated by Django 3.2.3 on 2021-06-14 10:58
-
-import uuid
-
-import django.db.models.deletion
-import enumfields.fields
-from django.conf import settings
-from django.db import migrations, models
-
-import arkindex.documents.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('documents', '0034_transcriptionentity_worker_version'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='CorpusExport',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('created', models.DateTimeField(auto_now_add=True)),
- ('updated', models.DateTimeField(auto_now=True)),
- ('state', enumfields.fields.EnumField(default='created', enum=arkindex.documents.models.CorpusExportState, max_length=10)),
- ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='exports', to='documents.corpus')),
- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='exports', to=settings.AUTH_USER_MODEL)),
- ],
- options={
- 'abstract': False,
- },
- ),
- ]
diff --git a/arkindex/documents/migrations/0036_elementpath_unique.py b/arkindex/documents/migrations/0036_elementpath_unique.py
deleted file mode 100644
index f2a52639ae6efe5d1048efb4d90f1c9e9f49a794..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0036_elementpath_unique.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Generated by Django 3.2.3 on 2021-06-23 10:27
-
-from django.db import migrations, models
-
-DEDUPLICATE_SQL = """
-DELETE FROM documents_elementpath WHERE id IN (
- SELECT id FROM (
- SELECT id, ROW_NUMBER() OVER (PARTITION BY element_id, path) > 1 AS is_duplicate
- FROM documents_elementpath
- ) AS a
- WHERE is_duplicate IS TRUE
-);
-"""
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0035_corpusexport'),
- ]
-
- operations = [
- migrations.RunSQL(
- DEDUPLICATE_SQL,
- reverse_sql=migrations.RunSQL.noop,
- ),
- migrations.AddConstraint(
- model_name='elementpath',
- constraint=models.UniqueConstraint(fields=('element', 'path'), name='unique_element_paths'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0037_metadata_numeric.py b/arkindex/documents/migrations/0037_metadata_numeric.py
deleted file mode 100644
index 6c81b9a5346a6e83df50086e3bee688f9c1d64dc..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0037_metadata_numeric.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Generated by Django 3.2.3 on 2021-06-25 10:03
-
-from django.db import migrations, models
-from django.db.models.expressions import RawSQL
-from django.db.models.functions import Cast
-
-from arkindex.documents.models import MetaType
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0036_elementpath_unique'),
- ]
-
- operations = [
- migrations.AddConstraint(
- model_name='metadata',
- constraint=models.CheckConstraint(
- check=~models.Q(type=MetaType.Numeric) | RawSQL(
- 'value::double precision IS NOT NULL',
- output_field=models.BooleanField(),
- params=()
- ),
- name='metadata_numeric_values'
- ),
- ),
- migrations.AddIndex(
- model_name='metadata',
- index=models.Index(
- Cast('value', output_field=models.FloatField()),
- condition=models.Q(type=MetaType.Numeric),
- name='metadata_numeric_index',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0038_element_zone_fields.py b/arkindex/documents/migrations/0038_element_zone_fields.py
deleted file mode 100644
index 3bf99401dae8b70784f18d995a2c0be94698ad75..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0038_element_zone_fields.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-19 12:14
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-import arkindex.project.fields
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0007_update_zone_constraint'),
- ('documents', '0037_metadata_numeric'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='image',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name='elements',
- to='images.image',
- ),
- ),
- migrations.AddField(
- model_name='element',
- name='polygon',
- field=arkindex.project.fields.LinearRingField(
- blank=True,
- null=True,
- srid=0,
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0039_migrate_zones.py b/arkindex/documents/migrations/0039_migrate_zones.py
deleted file mode 100644
index b780514f5271b019ece91e4be7a683bd5046343c..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0039_migrate_zones.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-19 12:14
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0038_element_zone_fields'),
- ]
-
- operations = [
- migrations.RunSQL("""
- UPDATE documents_element
- SET image_id = images_zone.image_id, polygon = images_zone.polygon
- FROM images_zone
- WHERE images_zone.id = documents_element.zone_id
- """),
- ]
diff --git a/arkindex/documents/migrations/0040_element_zone_constraints.py b/arkindex/documents/migrations/0040_element_zone_constraints.py
deleted file mode 100644
index 4255d9dd9ba0857595e08e5a5316873805e28212..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0040_element_zone_constraints.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-19 12:14
-
-from django.contrib.gis.db.models.functions import SnapToGrid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0039_migrate_zones'),
- ]
-
- operations = [
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(
- check=models.Q(image_id=None, polygon=None) | ~(models.Q(image_id=None) | models.Q(polygon=None)),
- name='element_image_and_polygon',
- ),
- ),
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(
- check=models.Q(polygon=None) | models.Q(polygon__isclosed=True),
- name='element_polygon_closed',
- ),
- ),
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(
- check=models.Q(polygon=None) | models.Q(polygon=SnapToGrid('polygon', 1)),
- name='element_polygon_integer_coordinates',
- ),
- ),
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(
- check=models.Q(polygon=None) | models.Q(polygon__numpoints__gte=4),
- name='element_polygon_size',
- ),
- ),
- migrations.RemoveField(
- model_name='element',
- name='zone',
- ),
- ]
diff --git a/arkindex/documents/migrations/0041_rotation.py b/arkindex/documents/migrations/0041_rotation.py
deleted file mode 100644
index 2493bd29d9711fc99a95c9421952bd8b7bea6929..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0041_rotation.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-21 10:26
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0040_element_zone_constraints'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='mirrored',
- field=models.BooleanField(
- default=False,
- help_text='Mirror the image along the vertical axis before rotating.',
- ),
- ),
- migrations.AddField(
- model_name='element',
- name='rotation_angle',
- field=models.SmallIntegerField(
- default=0,
- help_text='Clockwise rotation to apply to the image after cropping, in degrees.',
- validators=[django.core.validators.MaxValueValidator(359)],
- ),
- ),
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(
- check=models.Q(('rotation_angle__gte', 0), ('rotation_angle__lte', 359)),
- name='element_rotation_angle',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0042_transcription_entity_confidence.py b/arkindex/documents/migrations/0042_transcription_entity_confidence.py
deleted file mode 100644
index a20c9ef899a4ae4444cbfd46883537b30fd4b699..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0042_transcription_entity_confidence.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 3.2.6 on 2021-08-09 11:34
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0034_worker_run_config'),
- ('documents', '0041_rotation'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='transcriptionentity',
- name='confidence',
- field=models.FloatField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(1)]),
- ),
- ]
diff --git a/arkindex/documents/migrations/0043_element_creator.py b/arkindex/documents/migrations/0043_element_creator.py
deleted file mode 100644
index 79fb2b1ecbabbe7351d73b9137aadf94e86a98f3..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0043_element_creator.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 3.2.3 on 2021-10-12 09:46
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('documents', '0042_transcription_entity_confidence'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='creator',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='elements',
- to=settings.AUTH_USER_MODEL,
- ),
- ),
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(
- check=models.Q(creator_id=None) | models.Q(worker_version_id=None),
- name='element_creator_nand_worker_version',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0044_remove_metadata_index.py b/arkindex/documents/migrations/0044_remove_metadata_index.py
deleted file mode 100644
index f3f948bd6fb36afb129b42738402ac574e4c9cff..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0044_remove_metadata_index.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Generated by Django 3.2.3 on 2021-10-22 14:18
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0043_element_creator'),
- ]
-
- operations = [
- migrations.AlterModelOptions(
- name='metadata',
- options={'ordering': ('element', 'name', 'id')},
- ),
- migrations.AlterUniqueTogether(
- name='metadata',
- unique_together=set(),
- ),
- migrations.RemoveField(
- model_name='metadata',
- name='index',
- ),
- ]
diff --git a/arkindex/documents/migrations/0045_alter_element_rotation_angle.py b/arkindex/documents/migrations/0045_alter_element_rotation_angle.py
deleted file mode 100644
index 230d026eb348c4407e3f0e3fdd7f3a777d5aec27..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0045_alter_element_rotation_angle.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.2.3 on 2021-11-03 09:27
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0044_remove_metadata_index'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='element',
- name='rotation_angle',
- field=models.SmallIntegerField(default=0, help_text='Clockwise rotation to apply to the image after cropping, in degrees.', validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(359)]),
- ),
- ]
diff --git a/arkindex/documents/migrations/0046_transcription_orientation.py b/arkindex/documents/migrations/0046_transcription_orientation.py
deleted file mode 100644
index d790743787e258c023c3c43cc4a59b041754b498..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0046_transcription_orientation.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 3.2.6 on 2021-10-22 08:27
-
-import enumfields.fields
-from django.db import migrations
-
-import arkindex.documents.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0045_alter_element_rotation_angle'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='transcription',
- name='orientation',
- field=enumfields.fields.EnumField(default='horizontal-lr', enum=arkindex.documents.models.TextOrientation, max_length=50),
- ),
- ]
diff --git a/arkindex/documents/migrations/0047_elementpath_element_path_last.py b/arkindex/documents/migrations/0047_elementpath_element_path_last.py
deleted file mode 100644
index e52cadae8ecc3282089e098502547468a8ce6b49..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0047_elementpath_element_path_last.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.0.1 on 2022-01-05 16:29
-
-import django.db.models.expressions
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0046_transcription_orientation'),
- ]
-
- operations = [
- migrations.AddIndex(
- model_name='elementpath',
- index=models.Index(django.db.models.expressions.F('path__last'), name='element_path_last'),
- ),
- # After this index is created, we need to analyze the table again to make its query planner see it;
- # otherwise, it will keep its old way of querying and no performance improvement is made.
- migrations.RunSQL(
- "ANALYZE documents_elementpath",
- reverse_sql=migrations.RunSQL.noop,
- )
- ]
diff --git a/arkindex/documents/migrations/0048_empty_paths.py b/arkindex/documents/migrations/0048_empty_paths.py
deleted file mode 100644
index 4f462ca819a6c4bd85545f9c8bb2db5bbe33c7f6..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0048_empty_paths.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-18 14:31
-
-from django.db import migrations
-
-FORWARD_SQL = """
-INSERT INTO documents_elementpath (id, element_id, path, ordering)
-SELECT uuid_generate_v4(), e.id, ARRAY[]::uuid[], 0
-FROM documents_element e
-LEFT JOIN documents_elementpath p ON (p.element_id = e.id)
-WHERE p.id IS NULL;
-"""
-
-REVERSE_SQL = 'DELETE FROM documents_elementpath WHERE path = ARRAY[]::uuid[];'
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0047_elementpath_element_path_last'),
- ]
-
- operations = [
- migrations.RunSQL(FORWARD_SQL, reverse_sql=REVERSE_SQL)
- ]
diff --git a/arkindex/documents/migrations/0049_empty_path_constraints.py b/arkindex/documents/migrations/0049_empty_path_constraints.py
deleted file mode 100644
index 0fa647cc786e43529bf6ad945050fac3d840baa8..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0049_empty_path_constraints.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Generated by Django 4.0.1 on 2022-02-04 13:48
-
-from django.contrib.postgres.constraints import ExclusionConstraint
-from django.contrib.postgres.operations import BtreeGistExtension
-from django.db import migrations
-from django.db.models import Deferrable
-from django.db.models.functions import Least
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0048_empty_paths'),
- ]
-
- operations = [
- BtreeGistExtension(),
- migrations.AddConstraint(
- model_name='elementpath',
- constraint=ExclusionConstraint(
- name='unique_top_level',
- expressions=[
- ('element', '='),
- (Least('path__len', 1), '<>')
- ],
- deferrable=Deferrable.DEFERRED,
- )
- ),
- ]
diff --git a/arkindex/documents/migrations/0050_element_confidence.py b/arkindex/documents/migrations/0050_element_confidence.py
deleted file mode 100644
index 8f5eab08760b89107652c270dee4168c60cb838c..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0050_element_confidence.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 4.0.1 on 2022-05-06 09:25
-
-from django.core.validators import MaxValueValidator, MinValueValidator
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0049_empty_path_constraints'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='confidence',
- field=models.FloatField(
- blank=True,
- null=True,
- validators=[
- MinValueValidator(0),
- MaxValueValidator(1)
- ]
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0051_classification_worker_run.py b/arkindex/documents/migrations/0051_classification_worker_run.py
deleted file mode 100644
index 46bc623c062332114f0cb5618d465cb316490238..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0051_classification_worker_run.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Generated by Django 4.0.1 on 2022-05-11 13:58
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0047_workerversion_model_usage'),
- ('documents', '0050_element_confidence'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='classification',
- name='worker_run',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=models.DO_NOTHING,
- related_name='classifications',
- to='process.workerrun',
- ),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.CheckConstraint(
- check=models.Q(worker_version_id__isnull=False) | models.Q(worker_run_id__isnull=True),
- name='classification_worker_run_requires_worker_version',
- ),
- ),
- migrations.RemoveConstraint(
- model_name='classification',
- name='classification_unique_worker_version',
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(
- condition=models.Q(worker_run_id__isnull=True, worker_version_id__isnull=False),
- fields=('element', 'ml_class', 'worker_version'),
- name='classification_unique_worker_version',
- ),
- ),
- migrations.AddConstraint(
- model_name='classification',
- constraint=models.UniqueConstraint(
- condition=models.Q(worker_run_id__isnull=False),
- fields=('element', 'ml_class', 'worker_run'),
- name='classification_unique_worker_run',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0052_element_worker_run_and_more.py b/arkindex/documents/migrations/0052_element_worker_run_and_more.py
deleted file mode 100644
index 65acd6687a4a521916601c85f2cca66f6adfb6ec..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0052_element_worker_run_and_more.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.0.2 on 2022-05-13 14:35
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0047_workerversion_model_usage'),
- ('documents', '0051_classification_worker_run'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='element',
- name='worker_run',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='elements', to='process.workerrun'),
- ),
- migrations.AddConstraint(
- model_name='element',
- constraint=models.CheckConstraint(check=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False), _connector='OR'), name='element_worker_run_requires_worker_version'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0053_metadata_worker_run_and_more.py b/arkindex/documents/migrations/0053_metadata_worker_run_and_more.py
deleted file mode 100644
index 49e7520c791e6f14ff52c820ffbe806bd3fd9623..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0053_metadata_worker_run_and_more.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.0.2 on 2022-05-13 14:54
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0047_workerversion_model_usage'),
- ('documents', '0052_element_worker_run_and_more'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='metadata',
- name='worker_run',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='metadatas', to='process.workerrun'),
- ),
- migrations.AddConstraint(
- model_name='metadata',
- constraint=models.CheckConstraint(check=models.Q(('worker_version_id__isnull', False), ('worker_run_id__isnull', True), _connector='OR'), name='metadata_worker_run_requires_worker_version'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0054_transcription_worker_run.py b/arkindex/documents/migrations/0054_transcription_worker_run.py
deleted file mode 100644
index 6b401a75c29b022e25b223ad02c0062aa48b392d..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0054_transcription_worker_run.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Generated by Django 4.0.1 on 2022-05-13 12:13
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0047_workerversion_model_usage'),
- ('documents', '0053_metadata_worker_run_and_more'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='transcription',
- name='worker_run',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=models.DO_NOTHING,
- related_name='transcriptions',
- to='process.workerrun',
- ),
- ),
- migrations.AddConstraint(
- model_name='transcription',
- constraint=models.CheckConstraint(
- check=models.Q(worker_version_id__isnull=False) | models.Q(worker_run_id__isnull=True),
- name='transcription_worker_run_requires_worker_version',
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0055_entity_worker_run_and_more.py b/arkindex/documents/migrations/0055_entity_worker_run_and_more.py
deleted file mode 100644
index 7b6c3b8a8888fd1d45ae3174ff544881b046792c..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0055_entity_worker_run_and_more.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.0.2 on 2022-05-17 12:41
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0048_workerrun_model_version'),
- ('documents', '0054_transcription_worker_run'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='entity',
- name='worker_run',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='entities', to='process.workerrun'),
- ),
- migrations.AddConstraint(
- model_name='entity',
- constraint=models.CheckConstraint(check=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False), _connector='OR'), name='entity_worker_run_requires_worker_version'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0056_alter_transcriptionentity_unique_together_and_more.py b/arkindex/documents/migrations/0056_alter_transcriptionentity_unique_together_and_more.py
deleted file mode 100644
index fb4917827ddbb8ddceb4cd79290698df8dd02c06..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0056_alter_transcriptionentity_unique_together_and_more.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Generated by Django 4.0.2 on 2022-05-17 14:46
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0048_workerrun_model_version'),
- ('documents', '0055_entity_worker_run_and_more'),
- ]
-
- operations = [
- migrations.AlterUniqueTogether(
- name='transcriptionentity',
- unique_together=set(),
- ),
- migrations.AddField(
- model_name='transcriptionentity',
- name='worker_run',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='transcription_entities', to='process.workerrun'),
- ),
- migrations.AddConstraint(
- model_name='transcriptionentity',
- constraint=models.CheckConstraint(check=models.Q(('worker_version_id__isnull', False), ('worker_run_id__isnull', True), _connector='OR'), name='transcription_entity_worker_run_requires_worker_version'),
- ),
- migrations.AddConstraint(
- model_name='transcriptionentity',
- constraint=models.UniqueConstraint(condition=models.Q(('worker_version_id__isnull', True)), fields=('transcription', 'entity', 'offset', 'length'), name='transcription_entity_unique_manual'),
- ),
- migrations.AddConstraint(
- model_name='transcriptionentity',
- constraint=models.UniqueConstraint(condition=models.Q(('worker_run_id__isnull', True), ('worker_version_id__isnull', False)), fields=('transcription', 'entity', 'offset', 'length', 'worker_version'), name='transcription_entity_unique_worker_version'),
- ),
- migrations.AddConstraint(
- model_name='transcriptionentity',
- constraint=models.UniqueConstraint(condition=models.Q(('worker_run_id__isnull', False)), fields=('transcription', 'entity', 'offset', 'length', 'worker_run'), name='transcription_entity_unique_worker_run'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0057_entity_list_index.py b/arkindex/documents/migrations/0057_entity_list_index.py
deleted file mode 100644
index 434a8a439210077f3871b863cb94478dd7ae31e2..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0057_entity_list_index.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.5 on 2022-07-01 13:22
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0056_alter_transcriptionentity_unique_together_and_more'),
- ]
-
- operations = [
- migrations.AddIndex(
- model_name='entity',
- index=models.Index(fields=['corpus_id', 'name', 'id'], name='entity_list_index'),
- ),
- ]
diff --git a/arkindex/documents/migrations/0058_remove_corpus_repository.py b/arkindex/documents/migrations/0058_remove_corpus_repository.py
deleted file mode 100644
index 8c088e9e5fb17954f0546f5a04e4839245a91215..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0058_remove_corpus_repository.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-05 11:49
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0057_entity_list_index'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='corpus',
- name='repository',
- ),
- ]
diff --git a/arkindex/documents/migrations/0059_remove_corpus_thumbnail.py b/arkindex/documents/migrations/0059_remove_corpus_thumbnail.py
deleted file mode 100644
index ed17dae05651bfb9fc95583c64e5b81066e8302b..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0059_remove_corpus_thumbnail.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-30 10:16
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0058_remove_corpus_repository'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='corpus',
- name='thumbnail',
- ),
- ]
diff --git a/arkindex/documents/migrations/0060_rewrite_metadata_numeric.py b/arkindex/documents/migrations/0060_rewrite_metadata_numeric.py
deleted file mode 100644
index 8c785f6ee9bacc612dfff4f23723a4e718c939f4..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0060_rewrite_metadata_numeric.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 4.1.3 on 2022-12-05 16:16
-
-from django.db import migrations, models
-
-from arkindex.documents.models import MetaType
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0059_remove_corpus_thumbnail'),
- ]
-
- operations = [
- migrations.RemoveConstraint(
- model_name='metadata',
- name='metadata_numeric_values',
- ),
- migrations.AddConstraint(
- model_name='metadata',
- constraint=models.CheckConstraint(
- check=~models.Q(type=MetaType.Numeric) | models.Q(value__iregex=r'^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:E[+-]?\d+)?$'),
- name='metadata_numeric_values'
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0061_create_new_entitytypes.py b/arkindex/documents/migrations/0061_create_new_entitytypes.py
deleted file mode 100644
index 9877dee6695bb78368042f7aef2b52889bc7bcba..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0061_create_new_entitytypes.py
+++ /dev/null
@@ -1,94 +0,0 @@
-# Generated by Django 4.1.4 on 2023-01-09 16:51
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("documents", "0060_rewrite_metadata_numeric"),
- ]
-
- operations = [
- migrations.CreateModel(
- name="EntityType",
- fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4,
- editable=False,
- primary_key=True,
- serialize=False,
- ),
- ),
- ("name", models.CharField(max_length=250)),
- ("color", models.CharField(max_length=6, default='ff0000')),
- (
- "corpus",
- models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="entity_types",
- to="documents.corpus",
- ),
- ),
- ],
- options={
- "unique_together": {("name", "corpus")},
- },
- ),
- migrations.RenameField(
- model_name='entity',
- old_name='type',
- new_name='old_type',
- ),
- migrations.RenameField(
- model_name='entityrole',
- old_name='child_type',
- new_name='old_child_type',
- ),
- migrations.RenameField(
- model_name='entityrole',
- old_name='parent_type',
- new_name='old_parent_type',
- ),
- migrations.AddField(
- model_name="entity",
- name="type",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="entities",
- to="documents.entitytype",
- null=True,
- )
- ),
- migrations.AddField(
- model_name="entityrole",
- name="child_type",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="child_role",
- to="documents.entitytype",
- null=True,
- ),
- ),
- migrations.AddField(
- model_name="entityrole",
- name="parent_type",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="parent_role",
- to="documents.entitytype",
- null=True,
- ),
- ),
- migrations.AlterUniqueTogether(
- name="entityrole",
- unique_together={
- ("parent_name", "child_name", "parent_type", "child_type", "corpus")
- },
- ),
- ]
diff --git a/arkindex/documents/migrations/0062_migrate_entitytypes.py b/arkindex/documents/migrations/0062_migrate_entitytypes.py
deleted file mode 100644
index ff1963477687c79900716b63863270f436edf259..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0062_migrate_entitytypes.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("documents", "0061_create_new_entitytypes"),
- ]
-
- operations = [
- migrations.RunSQL(
- """
- INSERT INTO documents_entitytype (id, corpus_id, name, color)
- SELECT uuid_generate_v4(), corpus_id, name,
- CASE WHEN old_type = 'location' THEN '28b62c'
- WHEN old_type = 'date' THEN 'ff4136'
- WHEN old_type = 'person' THEN '3273dc'
- WHEN old_type = 'subject' THEN 'ffdd57'
- WHEN old_type = 'organization' THEN 'ea4aaa'
- WHEN old_type = 'misc' THEN '363636'
- WHEN old_type = 'number' THEN '00d1b2'
- ELSE 'ff0000' END
- FROM (
- SELECT DISTINCT ON (corpus_id, name) *
- FROM (
- SELECT
- corpus_id,
- COALESCE(metas->'subtype', old_type) AS name,
- old_type
- FROM documents_entity
- UNION ALL
- SELECT
- corpus_id,
- old_parent_type AS name,
- old_parent_type AS old_type
- FROM documents_entityrole
- UNION ALL
- SELECT
- corpus_id,
- old_child_type AS name,
- old_child_type AS old_type
- FROM documents_entityrole
- ) AS all_old_types
- ) AS old_types
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- migrations.RunSQL(
- """
- UPDATE documents_entity e
- SET type_id = t.id
- FROM documents_entitytype t
- WHERE t.corpus_id = e.corpus_id
- AND
- t.name = COALESCE(e.metas->'subtype', e.old_type)
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- migrations.RunSQL(
- """
- UPDATE documents_entityrole r
- SET child_type_id = t.id
- FROM documents_entitytype t
- WHERE t.corpus_id = r.corpus_id AND t.name = r.old_child_type
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- migrations.RunSQL(
- """
- UPDATE documents_entityrole r
- SET parent_type_id = t.id
- FROM documents_entitytype t
- WHERE t.corpus_id = r.corpus_id AND t.name = r.old_parent_type
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- ]
diff --git a/arkindex/documents/migrations/0063_remove_old_entitypes.py b/arkindex/documents/migrations/0063_remove_old_entitypes.py
deleted file mode 100644
index f28e0313f292da8377b1afa4d52e13ee6313cb60..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0063_remove_old_entitypes.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("documents", "0062_migrate_entitytypes"),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='entity',
- name='old_type'
- ),
- migrations.RemoveField(
- model_name='entityrole',
- name='old_child_type'
- ),
- migrations.RemoveField(
- model_name='entityrole',
- name='old_parent_type'
- ),
- migrations.RunSQL(
- """
- UPDATE documents_entity SET metas = metas - 'subtype'::text WHERE metas ? 'subtype'
- """,
- reverse_sql=migrations.RunSQL.noop,
- elidable=True,
- ),
- migrations.AlterField(
- model_name='entity',
- name='type',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="entities",
- to="documents.entitytype",
- )
- ),
- migrations.AlterField(
- model_name='entityrole',
- name='child_type',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="child_role",
- to="documents.entitytype",
- )
- ),
- migrations.AlterField(
- model_name='entityrole',
- name='parent_type',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="parent_role",
- to="documents.entitytype",
- )
- )
- ]
diff --git a/arkindex/documents/migrations/0064_alter_entity_type_alter_entityrole_child_type_and_more.py b/arkindex/documents/migrations/0064_alter_entity_type_alter_entityrole_child_type_and_more.py
deleted file mode 100644
index b57cf3d8c296b7c652debe17793443c9bc0be780..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0064_alter_entity_type_alter_entityrole_child_type_and_more.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Generated by Django 4.1.4 on 2023-02-01 10:46
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("documents", "0063_remove_old_entitypes"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="entity",
- name="type",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.DO_NOTHING,
- related_name="entities",
- to="documents.entitytype",
- ),
- ),
- migrations.AlterField(
- model_name="entityrole",
- name="child_type",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.DO_NOTHING,
- related_name="child_role",
- to="documents.entitytype",
- ),
- ),
- migrations.AlterField(
- model_name="entityrole",
- name="parent_type",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.DO_NOTHING,
- related_name="parent_role",
- to="documents.entitytype",
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0065_elementpath_constraints.py b/arkindex/documents/migrations/0065_elementpath_constraints.py
deleted file mode 100644
index 6f8ad212c60b860c5708f0bc3cbfbbe79c492046..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0065_elementpath_constraints.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Generated by Django 4.1.7 on 2023-04-26 14:12
-
-from django.contrib.postgres.constraints import ExclusionConstraint
-from django.contrib.postgres.fields import RangeOperators
-from django.db import migrations
-from django.db.models import Deferrable
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0064_alter_entity_type_alter_entityrole_child_type_and_more'),
- ]
-
- operations = [
- migrations.AddConstraint(
- model_name='elementpath',
- constraint=ExclusionConstraint(
- name='unique_element_orderings',
- expressions=[
- ('element', RangeOperators.EQUAL),
- ('path__last', RangeOperators.EQUAL),
- ('ordering', RangeOperators.NOT_EQUAL),
- ],
- deferrable=Deferrable.DEFERRED,
- ),
- ),
- migrations.AddConstraint(
- model_name='elementpath',
- constraint=ExclusionConstraint(
- name='unique_parent_orderings',
- expressions=[
- ('path__last', RangeOperators.EQUAL),
- ('ordering', RangeOperators.EQUAL),
- ('element', RangeOperators.NOT_EQUAL),
- ],
- deferrable=Deferrable.DEFERRED,
- ),
- ),
- ]
diff --git a/arkindex/documents/migrations/0066_element_type_implies_corpus.py b/arkindex/documents/migrations/0066_element_type_implies_corpus.py
deleted file mode 100644
index 44ddb9a4cd80034d796bc818e646b6407253663b..0000000000000000000000000000000000000000
--- a/arkindex/documents/migrations/0066_element_type_implies_corpus.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Generated by Django 4.1.7 on 2023-05-23 12:42
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0065_elementpath_constraints'),
- ]
-
- operations = [
- # This gives a hint to PostgreSQL's query planner that the element's corpus can be deduced from the type.
- # If you filter by a specific type ID, then the corpus ID will be constant;
- # all elements of the same type ID should be in the same corpus.
- #
- # This causes the query planner to create significantly better query plans when filtering by both corpus and type ID,
- # as it understands that filtering on both type and corpus is the same as only filtering on the type,
- # and the corpus filter will not reduce the planner's estimated row counts.
- #
- # On freshly created databases, this applies immediately. On existing databases, this will only have an effect after
- # an autovacuum completes or after an administrator runs `ANALYZE documents_element`.
- #
- # See https://gitlab.com/teklia/arkindex/backend/-/issues/1309#note_1401556847 for context.
-
- migrations.RunSQL(
- "CREATE STATISTICS element_type_implies_corpus (dependencies) ON type_id, corpus_id FROM documents_element",
- "DROP STATISTICS element_type_implies_corpus",
- elidable=False,
- ),
- ]
diff --git a/arkindex/images/migrations/0001_initial.py b/arkindex/images/migrations/0001_initial.py
index 3015034398a494be04172393204775e44c62bed4..7c36bac3b14a9d1c41ab0c368aef82822f2786da 100644
--- a/arkindex/images/migrations/0001_initial.py
+++ b/arkindex/images/migrations/0001_initial.py
@@ -1,48 +1,28 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import uuid
import django.db.models.deletion
import enumfields.fields
+from django.contrib.postgres.operations import CreateExtension
from django.db import migrations, models
import arkindex.project.aws
import arkindex.project.fields
-POLYGON_HASH_FORWARD_SQL = """
-ALTER TABLE images_zone
- ADD COLUMN IF NOT EXISTS polygon_hash VARCHAR(32)
- GENERATED ALWAYS AS (md5(polygon::varchar)) STORED;
-CREATE UNIQUE INDEX zone_unique_image_polygon ON images_zone (image_id, polygon_hash);
-"""
-
-POLYGON_HASH_REVERSE_SQL = """
-DROP INDEX IF EXISTS zone_unique_image_polygon;
-ALTER TABLE images_zone DROP COLUMN IF EXISTS polygon_hash;
-"""
-
-
-class PolygonField(models.Field):
- """
- Field to store a polygon; needs at least three set of points
- """
- def db_type(self, connection):
- return 'polygon'
-
class Migration(migrations.Migration):
initial = True
dependencies = [
- ('process', '0001_initial'),
]
operations = [
+ CreateExtension('postgis'),
migrations.CreateModel(
name='Image',
fields=[
- ('status', enumfields.fields.EnumField(default='unchecked', enum=arkindex.project.aws.S3FileStatus, max_length=50)),
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
@@ -50,7 +30,7 @@ class Migration(migrations.Migration):
('width', models.PositiveIntegerField(default=0)),
('height', models.PositiveIntegerField(default=0)),
('hash', arkindex.project.fields.MD5HashField(blank=True, max_length=32, null=True)),
- ('datafile', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='images', to='process.DataFile')),
+ ('status', enumfields.fields.EnumField(default='unchecked', enum=arkindex.project.aws.S3FileStatus, max_length=50)),
],
bases=(arkindex.project.aws.S3FileMixin, models.Model),
),
@@ -62,47 +42,28 @@ class Migration(migrations.Migration):
('url', arkindex.project.fields.StripSlashURLField(unique=True)),
('s3_bucket', models.SlugField(blank=True, db_index=False, max_length=63, null=True)),
('s3_region', models.SlugField(blank=True, db_index=False, max_length=63, null=True)),
+ ('s3_read_only_bucket', models.BooleanField(default=False, help_text='Disable the extra image checks that are normally made for servers with S3 buckets.')),
('max_width', models.PositiveIntegerField(blank=True, null=True)),
('max_height', models.PositiveIntegerField(blank=True, null=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
- ('validated', models.BooleanField(default=False)),
- ('read_only', models.BooleanField(default=False)),
],
),
- migrations.CreateModel(
- name='Zone',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('created', models.DateTimeField(auto_now_add=True)),
- ('updated', models.DateTimeField(auto_now=True)),
- ('polygon', PolygonField()),
- ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='zones', to='images.Image')),
- ],
+ migrations.AddConstraint(
+ model_name='imageserver',
+ constraint=models.CheckConstraint(check=models.Q(('s3_read_only_bucket', False), ('s3_bucket__isnull', False), _connector='OR'), name='s3_read_only_bucket_requires_s3_bucket'),
),
migrations.AddField(
model_name='image',
name='server',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to='images.ImageServer'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to='images.imageserver'),
),
- # When resetting migrations, replace the default AddConstraint on zone_unique_image_polygon with this RunSQL
- # Required to add the unique constraint on the polygon_hash field, which is invisible to Django
- # Otherwise, you will run into errors related to the requirement of an operator class for polygon on btree indexes.
- migrations.RunSQL(
- sql=POLYGON_HASH_FORWARD_SQL,
- reverse_sql=POLYGON_HASH_REVERSE_SQL,
- state_operations=[
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.UniqueConstraint(
- fields=('image', 'polygon'),
- name='zone_unique_image_polygon',
- ),
- ),
- ],
+ migrations.AddConstraint(
+ model_name='image',
+ constraint=models.CheckConstraint(check=models.Q(models.Q(('height__gt', 0), ('width__gt', 0)), models.Q(('status', arkindex.project.aws.S3FileStatus['Checked']), _negated=True), _connector='OR'), name='checked_image_dimensions'),
),
migrations.AlterUniqueTogether(
name='image',
- unique_together={('server', 'path'), ('server', 'hash')},
+ unique_together={('server', 'hash'), ('server', 'path')},
),
]
diff --git a/arkindex/images/migrations/0002_checked_dimensions.py b/arkindex/images/migrations/0002_checked_dimensions.py
deleted file mode 100644
index cca62ee91a3346028ecaa9ebcec1c5ae301bbfed..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0002_checked_dimensions.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 2.2.10 on 2020-05-13 15:17
-
-from django.db import migrations, models
-
-import arkindex.project.aws
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0001_initial'),
- ]
-
- operations = [
- migrations.RunSQL(
- sql="""
- UPDATE images_image
- SET status = 'unchecked'
- WHERE status = 'checked' AND (width = 0 OR height = 0);
- """,
- reverse_sql=migrations.RunSQL.noop,
- ),
- migrations.AddConstraint(
- model_name='image',
- constraint=models.CheckConstraint(
- check=models.Q(
- models.Q(('height__gt', 0), ('width__gt', 0)),
- models.Q(_negated=True, status=arkindex.project.aws.S3FileStatus('checked')),
- _connector='OR',
- ),
- name='checked_image_dimensions',
- ),
- ),
- ]
diff --git a/arkindex/images/migrations/0003_remove_image_datafile.py b/arkindex/images/migrations/0003_remove_image_datafile.py
deleted file mode 100644
index 31fe8363edc6acaf3ceaf786da7c994090736882..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0003_remove_image_datafile.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-11 12:49
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0002_checked_dimensions'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='image',
- name='datafile',
- ),
- ]
diff --git a/arkindex/images/migrations/0004_polygon_postgis.py b/arkindex/images/migrations/0004_polygon_postgis.py
deleted file mode 100644
index 54bc4dcd2ed3346889048fa499b85e12319fabe2..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0004_polygon_postgis.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Generated by Django 2.2.13 on 2020-07-29 12:24
-
-from django.contrib.postgres.operations import CreateExtension
-from django.db import migrations
-
-from arkindex.project.fields import LinearRingField
-from arkindex.project.gis import SimplifyPreserveTopology
-
-POLYGON_HASH_FORWARD_SQL = """
-DROP INDEX IF EXISTS zone_unique_image_polygon;
-ALTER TABLE images_zone DROP COLUMN IF EXISTS polygon_hash;
-"""
-
-POLYGON_HASH_REVERSE_SQL = """
-ALTER TABLE images_zone
- ADD COLUMN IF NOT EXISTS polygon_hash VARCHAR(32)
- GENERATED ALWAYS AS (md5(polygon::varchar)) STORED;
-CREATE UNIQUE INDEX zone_unique_image_polygon ON images_zone (image_id, polygon_hash);
-"""
-
-# Experiments with production data has shown a tolerance of 15 is enough to simplify all polygons;
-# we will try with tolerances from 1 to 20 to be safe.
-MAX_TOLERANCE = 20
-
-
-def simplify_polygons(apps, schema_editor):
- Zone = apps.get_model('images', 'Zone')
- if not Zone.objects.exists():
- return
-
- print('Looking for polygons exceeding the maximum allowed size…')
- count = Zone.objects.filter(polygon__memsize__gt=2664).count()
- if not count:
- return
-
- for tolerance in range(1, MAX_TOLERANCE - 1):
- print(f'Simplifying {count} polygons with tolerance {tolerance}…')
- Zone.objects \
- .filter(polygon__memsize__gt=2664) \
- .update(polygon=SimplifyPreserveTopology('polygon', tolerance))
-
- count = Zone.objects.filter(polygon__memsize__gt=2664).count()
- if not count:
- print('Simplification successful!')
- return
-
- raise ValueError('{count} existing polygons could not be simplified with a tolerance of {MAX_TOLERANCE}.')
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0003_remove_image_datafile'),
- ]
-
- operations = [
- CreateExtension('postgis'),
-
- # In images.0001, we made Django believe we managed to get a unique constraint
- # on (image_id, polygon), so it now believes it does not need to change it.
- # However, the previous unique constraint used a hack with polygon_hash, whereas
- # the PostGIS geometry type allows an actual unique constraint without a generated column.
- # Therefore, we first drop the previous constraint, and will recreate it differently later.
- migrations.RunSQL(
- POLYGON_HASH_FORWARD_SQL,
- reverse_sql=POLYGON_HASH_REVERSE_SQL,
- state_operations=[
- migrations.RemoveConstraint(
- model_name='zone',
- name='zone_unique_image_polygon'
- )
- ],
- ),
-
- # Django is unable to migrate from a polygon to a geometry(LINESTRING)
- # since it would expect geometry(POLYGON). A PostGIS polygon has multiple rings,
- # but we can only handle one ring, so a closed LineString is what we want.
- # To update properly, we apply ST_Normalize and ST_RemoveRepeatedPoints
- # (Polygon.reorder but for PostGIS), then we add potentially missing final points
- # (ABCD instead of ABCDA), which would make a LineString be left open.
- migrations.RunSQL(
- sql="""
- ALTER TABLE images_zone
- ALTER COLUMN polygon
- TYPE geometry(LINESTRING, 0)
- USING ST_RemoveRepeatedPoints(ST_ExteriorRing(ST_Normalize(polygon::geometry(POLYGON, 0))));
-
- UPDATE images_zone
- SET polygon = ST_AddPoint(polygon, ST_StartPoint(polygon))
- WHERE NOT ST_IsClosed(polygon);
- """,
- reverse_sql="""
- ALTER TABLE images_zone
- ALTER COLUMN polygon
- TYPE polygon
- USING pclose(polygon::path)::polygon
- """,
- state_operations=[
- migrations.AlterField(
- model_name='zone',
- name='polygon',
- field=LinearRingField(srid=0),
- )
- ]
- ),
-
- migrations.RunPython(
- simplify_polygons,
- reverse_code=migrations.RunPython.noop,
- ),
-
- # ST_Normalize, the extra ending point and simplifications can lead to a small portion
- # of the zones being duplicated due to that applying a more effective reordering than we did;
- # this can require running `arkindex deduplicate_zones` to manually deduplicate zones
- # before adding constraints.
- # Adding the constraints is also done in another migration due to 'pending trigger events'
- # on images_zone after the deduplication and simplification.
- ]
diff --git a/arkindex/images/migrations/0005_polygon_index.py b/arkindex/images/migrations/0005_polygon_index.py
deleted file mode 100644
index 6b59929218f15e8cff0dcd6110e72781f971744d..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0005_polygon_index.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Generated by Django 2.2.13 on 2020-07-30 13:54
-
-from django.contrib.gis.db.models.functions import SnapToGrid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0004_polygon_postgis'),
- ]
-
- operations = [
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.CheckConstraint(
- check=models.Q(polygon__memsize__lte=2664),
- name='zone_polygon_size',
- ),
- ),
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.CheckConstraint(
- check=models.Q(polygon__isclosed=True),
- name='zone_polygon_closed',
- ),
- ),
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.CheckConstraint(
- check=models.Q(polygon=SnapToGrid('polygon', 1)),
- name='zone_polygon_integer_coordinates',
- ),
- ),
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.UniqueConstraint(
- fields=('image', 'polygon'),
- name='zone_unique_image_polygon',
- ),
- ),
- ]
diff --git a/arkindex/images/migrations/0006_polygon_min_size.py b/arkindex/images/migrations/0006_polygon_min_size.py
deleted file mode 100644
index 5c232c5cb21f724404319ea1fc45987a16c8e9ea..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0006_polygon_min_size.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Generated by Django 3.1.3 on 2021-01-04 11:04
-
-from django.db import migrations, models
-
-
-def remove_small_polygons(apps, schema_editor):
- """
- Removes polygons with less than 4 points.
- """
- Element = apps.get_model('documents', 'Element')
- Zone = apps.get_model('images', 'Zone')
-
- zone_ids = list(Zone.objects.using('default').filter(polygon__memsize__lt=96).values_list('id', flat=True))
- if not zone_ids:
- return
-
- # Import the ElementQuerySet to access the trash() method
- from arkindex.documents.managers import ElementQuerySet
- ElementQuerySet(Element).filter(zone_id__in=zone_ids).trash()
-
- Zone.objects.filter(id__in=zone_ids)._raw_delete(using='default')
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0005_polygon_index'),
- ]
- atomic = False
-
- operations = [
- migrations.RemoveConstraint(
- model_name='zone',
- name='zone_polygon_size',
- ),
- migrations.RunPython(
- remove_small_polygons,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.CheckConstraint(
- check=models.Q(polygon__memsize__gte=96, polygon__memsize__lte=2664),
- name='zone_polygon_size'
- ),
- ),
- ]
diff --git a/arkindex/images/migrations/0007_update_zone_constraint.py b/arkindex/images/migrations/0007_update_zone_constraint.py
deleted file mode 100644
index c84ddf1692fdcb0bf47b2d98fbc2e92eb0be9c8f..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0007_update_zone_constraint.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Generated by Django 3.1.8 on 2021-04-26 13:32
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0006_polygon_min_size'),
- ]
-
- operations = [
- migrations.RemoveConstraint(
- model_name='zone',
- name='zone_polygon_size',
- ),
- migrations.AddConstraint(
- model_name='zone',
- constraint=models.CheckConstraint(check=models.Q(('polygon__numpoints__gte', 4), ('polygon__numpoints__lte', 300)), name='zone_polygon_size'),
- ),
- migrations.AlterModelOptions(
- name='zone',
- options={'base_manager_name': 'objects'},
- ),
- ]
diff --git a/arkindex/images/migrations/0008_delete_zone.py b/arkindex/images/migrations/0008_delete_zone.py
deleted file mode 100644
index cb2f4f771994b850f87717bcfed9145f6e2bb901..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0008_delete_zone.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-19 12:14
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0040_element_zone_constraints'),
- ('images', '0007_update_zone_constraint'),
- ]
-
- operations = [
- migrations.DeleteModel(
- name='Zone',
- ),
- ]
diff --git a/arkindex/images/migrations/0009_remove_imageserver_validated.py b/arkindex/images/migrations/0009_remove_imageserver_validated.py
deleted file mode 100644
index 2ee2bcefac44c0db9b89fdafc451a0575f4e3f10..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0009_remove_imageserver_validated.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.7 on 2022-10-12 14:39
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0008_delete_zone'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='imageserver',
- name='validated',
- ),
- ]
diff --git a/arkindex/images/migrations/0010_rename_imageserver_readonly.py b/arkindex/images/migrations/0010_rename_imageserver_readonly.py
deleted file mode 100644
index 70ac8072850cfd2c42ff0f06280efc75050979a6..0000000000000000000000000000000000000000
--- a/arkindex/images/migrations/0010_rename_imageserver_readonly.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# Generated by Django 4.0.7 on 2022-10-14 13:27
-
-from django.db import migrations, models
-
-
-def disable_read_only_without_bucket(apps, schema_editor):
- """
- Turn off s3_read_only_bucket on servers without S3 buckets
- to ensure that the new check constraint is respected.
- """
- ImageServer = apps.get_model('images', 'ImageServer')
- ImageServer.objects.filter(s3_bucket__isnull=True).update(s3_read_only_bucket=False)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('images', '0009_remove_imageserver_validated'),
- ]
-
- operations = [
- migrations.RenameField(
- model_name='imageserver',
- old_name='read_only',
- new_name='s3_read_only_bucket',
- ),
- migrations.AlterField(
- model_name='imageserver',
- name='s3_read_only_bucket',
- field=models.BooleanField(
- default=False,
- help_text='Disable the extra image checks that are normally made for servers with S3 buckets.',
- ),
- ),
- migrations.RunPython(
- disable_read_only_without_bucket,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AddConstraint(
- model_name='imageserver',
- constraint=models.CheckConstraint(
- check=models.Q(s3_read_only_bucket=False) | models.Q(s3_bucket__isnull=False),
- name='s3_read_only_bucket_requires_s3_bucket'
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0001_initial.py b/arkindex/ponos/migrations/0001_initial.py
index 083fb7c137f16e0e78805b6fc26612aac5358cbe..fd5bdd16a3e7ada7826e36d1b6773e46d868c21f 100644
--- a/arkindex/ponos/migrations/0001_initial.py
+++ b/arkindex/ponos/migrations/0001_initial.py
@@ -1,80 +1,170 @@
-# Generated by Django 2.1.4 on 2018-12-20 16:03
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import uuid
+import django.contrib.postgres.fields.hstore
+import django.core.validators
import django.db.models.deletion
import enumfields.fields
+from django.contrib.postgres.operations import HStoreExtension
from django.db import migrations, models
+import arkindex.ponos.keys
import arkindex.ponos.models
+import arkindex.ponos.validators
+import arkindex.project.fields
class Migration(migrations.Migration):
initial = True
- dependencies = []
+ dependencies = [
+ ]
operations = [
+ HStoreExtension(),
+ migrations.CreateModel(
+ name='Agent',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('public_key', models.TextField()),
+ ('hostname', models.SlugField(db_index=False, max_length=64)),
+ ('cpu_cores', models.PositiveSmallIntegerField(validators=[django.core.validators.MinValueValidator(1)])),
+ ('cpu_frequency', models.BigIntegerField(validators=[django.core.validators.MinValueValidator(1)])),
+ ('ram_total', models.BigIntegerField(validators=[django.core.validators.MinValueValidator(1)])),
+ ('cpu_load', models.FloatField(blank=True, null=True)),
+ ('ram_load', models.FloatField(blank=True, null=True)),
+ ('last_ping', models.DateTimeField(editable=False)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Artifact',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+ ('path', models.CharField(max_length=500)),
+ ('size', models.BigIntegerField(validators=[django.core.validators.MinValueValidator(1), arkindex.ponos.validators.MaxValueValidator(arkindex.ponos.models.artifact_max_size)])),
+ ('content_type', models.CharField(default='application/octet-stream', max_length=250)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ],
+ options={
+ 'ordering': ('task', 'path'),
+ },
+ ),
+ migrations.CreateModel(
+ name='Farm',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=250)),
+ ('seed', models.CharField(default=arkindex.ponos.models.generate_seed, max_length=64, unique=True, validators=[django.core.validators.RegexValidator('^[0-9a-f]{64}$')])),
+ ],
+ ),
+ migrations.CreateModel(
+ name='GPU',
+ fields=[
+ ('id', models.UUIDField(primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=250)),
+ ('index', models.PositiveIntegerField()),
+ ('ram_total', models.BigIntegerField(validators=[django.core.validators.MinValueValidator(1)])),
+ ('agent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='gpus', to='ponos.agent')),
+ ],
+ ),
migrations.CreateModel(
- name="Task",
+ name='Secret',
fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4, primary_key=True, serialize=False
- ),
- ),
- ("run", models.PositiveIntegerField()),
- ("depth", models.PositiveIntegerField()),
- ("slug", models.CharField(max_length=250)),
- (
- "state",
- enumfields.fields.EnumField(
- default="unscheduled", enum=arkindex.ponos.models.State, max_length=20
- ),
- ),
- ("container", models.CharField(blank=True, max_length=64, null=True)),
- ("created", models.DateTimeField(auto_now_add=True)),
- ("updated", models.DateTimeField(auto_now=True)),
- (
- "parent",
- models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name="children",
- to="ponos.Task",
- ),
- ),
+ ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=250, unique=True)),
+ ('nonce', models.BinaryField(default=arkindex.ponos.keys.gen_nonce, max_length=16)),
+ ('content', models.BinaryField(editable=True)),
],
- options={"ordering": ("workflow", "run", "depth", "slug")},
),
migrations.CreateModel(
- name="Workflow",
+ name='Workflow',
fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4, primary_key=True, serialize=False
- ),
- ),
- ("recipe", models.TextField()),
- ("created", models.DateTimeField(auto_now_add=True)),
- ("updated", models.DateTimeField(auto_now=True)),
+ ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('finished', models.DateTimeField(blank=True, null=True)),
+ ('farm', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='workflows', to='ponos.farm')),
],
+ options={
+ 'ordering': ('-updated',),
+ },
+ ),
+ migrations.CreateModel(
+ name='Task',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+ ('run', models.PositiveIntegerField()),
+ ('depth', models.PositiveIntegerField()),
+ ('slug', models.CharField(max_length=250, validators=[django.core.validators.MinLengthValidator(1)])),
+ ('priority', models.PositiveIntegerField(default=10)),
+ ('state', enumfields.fields.EnumField(default='unscheduled', enum=arkindex.ponos.models.State, max_length=20)),
+ ('tags', arkindex.project.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, default=list, size=16)),
+ ('image', models.CharField(max_length=250)),
+ ('shm_size', models.CharField(blank=True, editable=False, max_length=80, null=True)),
+ ('command', models.TextField(blank=True, null=True)),
+ ('env', django.contrib.postgres.fields.hstore.HStoreField(default=dict)),
+ ('has_docker_socket', models.BooleanField(default=False)),
+ ('requires_gpu', models.BooleanField(default=False)),
+ ('container', models.CharField(blank=True, max_length=64, null=True)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('expiry', models.DateTimeField(default=arkindex.ponos.models.expiry_default)),
+ ('extra_files', django.contrib.postgres.fields.hstore.HStoreField(default=dict)),
+ ('token', models.CharField(default=arkindex.ponos.models.task_token_default, max_length=52, unique=True)),
+ ('agent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tasks', to='ponos.agent')),
+ ('gpu', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tasks', to='ponos.gpu')),
+ ('image_artifact', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tasks_using_image', to='ponos.artifact')),
+ ('parents', models.ManyToManyField(related_name='children', to='ponos.task')),
+ ('workflow', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='ponos.workflow')),
+ ],
+ options={
+ 'ordering': ('workflow', 'run', 'depth', 'slug'),
+ },
),
migrations.AddField(
- model_name="task",
- name="workflow",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="tasks",
- to="ponos.Workflow",
- ),
+ model_name='artifact',
+ name='task',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='artifacts', to='ponos.task'),
+ ),
+ migrations.AddField(
+ model_name='agent',
+ name='farm',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ponos.farm'),
+ ),
+ migrations.CreateModel(
+ name='AgentUser',
+ fields=[
+ ],
+ options={
+ 'proxy': True,
+ 'indexes': [],
+ 'constraints': [],
+ },
+ bases=('ponos.agent',),
+ ),
+ migrations.AddConstraint(
+ model_name='workflow',
+ constraint=models.CheckConstraint(check=models.Q(('finished', None), ('finished__gte', models.F('created')), _connector='OR'), name='ponos_workflow_finished_after_created'),
+ ),
+ migrations.AddConstraint(
+ model_name='task',
+ constraint=models.UniqueConstraint(models.F('gpu'), condition=models.Q(('state__in', (arkindex.ponos.models.State['Unscheduled'], arkindex.ponos.models.State['Pending'], arkindex.ponos.models.State['Running'], arkindex.ponos.models.State['Stopping']))), name='unique_gpu_on_active_tasks'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='task',
+ unique_together={('workflow', 'run', 'slug')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='gpu',
+ unique_together={('agent_id', 'index')},
),
migrations.AlterUniqueTogether(
- name="task",
- unique_together={("workflow", "run", "slug")},
+ name='artifact',
+ unique_together={('task', 'path')},
),
]
diff --git a/arkindex/ponos/migrations/0002_recipe_validator.py b/arkindex/ponos/migrations/0002_recipe_validator.py
deleted file mode 100644
index 6b971f659dd639eb02696bbd24739db503862ca4..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0002_recipe_validator.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 2.1.7 on 2019-06-24 08:57
-
-from django.db import migrations, models
-
-
-def recipe_validator(value) -> None:
- """
- This previously was a validator for YAML recipes.
- Since those recipes have now been removed, this function is left here only so that migrations remain consistent.
- """
- pass
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0001_initial"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="workflow",
- name="recipe",
- field=models.TextField(validators=[recipe_validator]),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0003_no_empty_slugs.py b/arkindex/ponos/migrations/0003_no_empty_slugs.py
deleted file mode 100644
index 6f3d1978d60b9bf6b50d5242860c1227dd2a8f7b..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0003_no_empty_slugs.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Generated by Django 2.1.7 on 2019-06-24 09:05
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0002_recipe_validator"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="task",
- name="slug",
- field=models.CharField(
- max_length=250,
- validators=[django.core.validators.MinLengthValidator(1)],
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0004_agent.py b/arkindex/ponos/migrations/0004_agent.py
deleted file mode 100644
index 26a3a3a84600be534045edd81f9a9e402caf08f5..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0004_agent.py
+++ /dev/null
@@ -1,76 +0,0 @@
-# Generated by Django 2.1.7 on 2019-06-05 08:48
-
-import uuid
-
-import django.core.validators
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0003_no_empty_slugs"),
- ]
-
- operations = [
- migrations.CreateModel(
- name="Agent",
- fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4, primary_key=True, serialize=False
- ),
- ),
- ("created", models.DateTimeField(auto_now_add=True)),
- ("updated", models.DateTimeField(auto_now=True)),
- (
- "token",
- models.CharField(
- max_length=250,
- unique=True,
- validators=[
- django.core.validators.RegexValidator(r"^[0-9a-f]{64}$")
- ],
- ),
- ),
- ("hostname", models.SlugField(db_index=False, max_length=64)),
- (
- "cpu_cores",
- models.PositiveSmallIntegerField(
- validators=[django.core.validators.MinValueValidator(1)],
- ),
- ),
- (
- "cpu_frequency",
- models.BigIntegerField(
- validators=[django.core.validators.MinValueValidator(1)],
- ),
- ),
- ("gpu_names", models.TextField()),
- ("gpu_count", models.PositiveSmallIntegerField()),
- ],
- ),
- migrations.AlterModelOptions(
- name="workflow",
- options={"ordering": ("-updated",)},
- ),
- migrations.AddField(
- model_name="task",
- name="agent",
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="tasks",
- to="ponos.Agent",
- ),
- ),
- migrations.CreateModel(
- name="AgentUser",
- fields=[],
- options={"proxy": True, "indexes": []},
- bases=("ponos.agent",),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0005_gpu_names_blank.py b/arkindex/ponos/migrations/0005_gpu_names_blank.py
deleted file mode 100644
index 5a33f4aea40121a12a35b110081360547a31e583..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0005_gpu_names_blank.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.1.7 on 2019-07-02 08:00
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0004_agent"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="agent",
- name="gpu_names",
- field=models.TextField(blank=True, null=True),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0006_add_parents.py b/arkindex/ponos/migrations/0006_add_parents.py
deleted file mode 100644
index 5d9f103262922647fdf4533e376f24184b95eb93..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0006_add_parents.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 2.1.7 on 2019-06-21 09:31
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0005_gpu_names_blank"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="parents",
- field=models.ManyToManyField(
- related_name="children", to="ponos.Task", symmetrical=False
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0007_migrate_parents.py b/arkindex/ponos/migrations/0007_migrate_parents.py
deleted file mode 100644
index d3d630314886714c8edd99b90d8ce35bc008612a..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0007_migrate_parents.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Generated by Django 2.1.7 on 2019-06-21 09:31
-
-from django.db import migrations
-from django.db.models import Count
-
-
-def parent_to_parents(apps, schema_editor):
- db_alias = schema_editor.connection.alias
- Task = apps.get_model("ponos", "Task")
- TaskParent = Task.parents.through
-
- new_parents = []
- for task in (
- Task.objects.using(db_alias).exclude(parent=None).only("id", "parent_id")
- ):
- new_parents.append(
- TaskParent(
- from_task_id=task.id,
- to_task_id=task.parent_id,
- )
- )
- TaskParent.objects.using(db_alias).bulk_create(new_parents)
-
-
-def parents_to_parent(apps, schema_editor):
- db_alias = schema_editor.connection.alias
- Task = apps.get_model("ponos", "Task")
- assert (
- not Task.objects.using(db_alias)
- .annotate(parents_count=Count("parents"))
- .filter(parents_count__gt=1)
- .exists()
- ), "Cannot migrate task with multiple parents backwards"
-
- for task in Task.objects.using(db_alias).filter(parents__isnull=False):
- task.parent = task.parents.get()
- task.save()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0006_add_parents"),
- ]
-
- operations = [
- migrations.RunPython(
- parent_to_parents,
- parents_to_parent,
- ),
- ]
diff --git a/arkindex/ponos/migrations/0008_remove_parent.py b/arkindex/ponos/migrations/0008_remove_parent.py
deleted file mode 100644
index 396fc3dedc3cdfb3e46e07a172997e6efc1265e5..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0008_remove_parent.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 2.1.7 on 2019-06-21 09:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0007_migrate_parents"),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name="task",
- name="parent",
- ),
- ]
diff --git a/arkindex/ponos/migrations/0009_tags.py b/arkindex/ponos/migrations/0009_tags.py
deleted file mode 100644
index a378c6a50d0ab60d37d74c82836f6e65ef01c61c..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0009_tags.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 2.1.7 on 2019-07-10 14:20
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0008_remove_parent"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="agent",
- name="tags",
- field=models.CharField(default="", max_length=250),
- ),
- migrations.AddField(
- model_name="task",
- name="tags",
- field=models.CharField(default="", max_length=250),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0010_farm.py b/arkindex/ponos/migrations/0010_farm.py
deleted file mode 100644
index 8f01d0d329e777a2971bea8a9c64ecdc914e19c2..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0010_farm.py
+++ /dev/null
@@ -1,91 +0,0 @@
-# Generated by Django 2.1.7 on 2019-07-03 14:07
-
-import uuid
-
-import django.core.validators
-import django.db.models.deletion
-from django.db import migrations, models
-
-import arkindex.ponos.models
-
-
-def default_farm(apps, schema_editor):
- db_alias = schema_editor.connection.alias
- Agent = apps.get_model("ponos", "Agent")
- if not Agent.objects.using(db_alias).exists():
- return
-
- Farm = apps.get_model("ponos", "Farm")
- default_farm = Farm.objects.using(db_alias).create(name="default")
- Agent.objects.using(db_alias).all().update(farm=default_farm)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0009_tags"),
- ]
-
- operations = [
- migrations.CreateModel(
- name="Farm",
- fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4,
- primary_key=True,
- serialize=False,
- ),
- ),
- (
- "name",
- models.CharField(
- max_length=250,
- ),
- ),
- (
- "seed",
- models.CharField(
- default=arkindex.ponos.models.generate_seed,
- max_length=64,
- unique=True,
- validators=[
- django.core.validators.RegexValidator("^[0-9a-f]{64}$")
- ],
- ),
- ),
- ],
- ),
- migrations.AddField(
- model_name="agent",
- name="farm",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.PROTECT,
- to="ponos.Farm",
- blank=True,
- null=True,
- ),
- ),
- migrations.RunPython(
- default_farm,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AlterField(
- model_name="agent",
- name="farm",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.PROTECT,
- to="ponos.Farm",
- ),
- ),
- migrations.AddField(
- model_name="agent",
- name="public_key",
- field=models.TextField(
- default="",
- ),
- preserve_default=False,
- ),
- ]
diff --git a/arkindex/ponos/migrations/0011_remove_agent_token.py b/arkindex/ponos/migrations/0011_remove_agent_token.py
deleted file mode 100644
index 6000d2cf471a709a1ed861f126fc47203ed9d19f..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0011_remove_agent_token.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 2.1.7 on 2019-07-04 13:34
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0010_farm"),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name="agent",
- name="token",
- ),
- ]
diff --git a/arkindex/ponos/migrations/0012_advanced_tags.py b/arkindex/ponos/migrations/0012_advanced_tags.py
deleted file mode 100644
index 073a13d5f28035b7a3f26e09a352fbe095d9e25d..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0012_advanced_tags.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 2.2.5 on 2019-09-04 13:34
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0011_remove_agent_token"),
- ]
-
- operations = [
- migrations.RenameField(
- model_name="agent",
- old_name="tags",
- new_name="include_tags",
- ),
- migrations.AddField(
- model_name="agent",
- name="exclude_tags",
- field=models.CharField(default="", max_length=250),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0013_artifact.py b/arkindex/ponos/migrations/0013_artifact.py
deleted file mode 100644
index 8f212321b4434e15296111680da3d9aec65c4067..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0013_artifact.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Generated by Django 2.2.12 on 2020-05-27 09:29
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0012_advanced_tags"),
- ]
-
- operations = [
- migrations.CreateModel(
- name="Artifact",
- fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4, primary_key=True, serialize=False
- ),
- ),
- ("path", models.CharField(max_length=500)),
- ("size", models.PositiveIntegerField()),
- (
- "content_type",
- models.CharField(
- default="application/octet-stream", max_length=250
- ),
- ),
- (
- "task",
- models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="artifacts",
- to="ponos.Task",
- ),
- ),
- ("created", models.DateTimeField(auto_now_add=True)),
- ("updated", models.DateTimeField(auto_now=True)),
- ],
- options={
- "ordering": ("task", "path"),
- "unique_together": {("task", "path")},
- },
- ),
- ]
diff --git a/arkindex/ponos/migrations/0014_modify_task_model.py b/arkindex/ponos/migrations/0014_modify_task_model.py
deleted file mode 100644
index eb07679696555688f37801317dac2c0cb6697d50..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0014_modify_task_model.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by Django 2.2.12 on 2020-06-05 08:53
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0013_artifact"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="command",
- field=models.TextField(null=True),
- ),
- migrations.AddField(
- model_name="task",
- name="env",
- field=models.JSONField(null=True),
- ),
- migrations.AddField(
- model_name="task",
- name="image",
- field=models.CharField(null=True, max_length=250),
- ),
- migrations.AlterField(
- model_name="task",
- name="image",
- field=models.CharField(max_length=250),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0015_task_has_docker_socket.py b/arkindex/ponos/migrations/0015_task_has_docker_socket.py
deleted file mode 100644
index de50d041674c133fdc9d393a2528c7d2bef95102..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0015_task_has_docker_socket.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.2.12 on 2020-06-11 08:30
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0014_modify_task_model"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="has_docker_socket",
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0016_task_image_artifact.py b/arkindex/ponos/migrations/0016_task_image_artifact.py
deleted file mode 100644
index aa3cf3ea307e44e8fc1805e56e5754ad55864980..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0016_task_image_artifact.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 2.2.13 on 2020-07-07 08:42
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0015_task_has_docker_socket"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="image_artifact",
- field=models.ForeignKey(
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="tasks_using_image",
- to="ponos.Artifact",
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0017_new_jsonfield.py b/arkindex/ponos/migrations/0017_new_jsonfield.py
deleted file mode 100644
index 24e2875da652649b89095790f04cd2cea926923f..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0017_new_jsonfield.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1 on 2020-08-10 14:59
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0016_task_image_artifact"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="task",
- name="env",
- field=models.JSONField(null=True),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0018_auto_20200814_0818.py b/arkindex/ponos/migrations/0018_auto_20200814_0818.py
deleted file mode 100644
index 6ce079bf148d6e03cfb4f8a4b8c2620938750686..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0018_auto_20200814_0818.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1 on 2020-08-14 08:18
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0017_new_jsonfield"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="artifact",
- name="size",
- field=models.PositiveBigIntegerField(),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0019_secret.py b/arkindex/ponos/migrations/0019_secret.py
deleted file mode 100644
index bb33c07f76f45428bf23c489f5e4dfffd06404f5..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0019_secret.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 3.1 on 2020-09-24 14:12
-
-import uuid
-
-from django.db import migrations, models
-
-import arkindex.ponos.keys
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0018_auto_20200814_0818"),
- ]
-
- operations = [
- migrations.CreateModel(
- name="Secret",
- fields=[
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4, primary_key=True, serialize=False
- ),
- ),
- ("name", models.CharField(max_length=250, unique=True)),
- (
- "nonce",
- models.BinaryField(default=arkindex.ponos.keys.gen_nonce, max_length=16),
- ),
- ("content", models.BinaryField(editable=True)),
- ],
- ),
- ]
diff --git a/arkindex/ponos/migrations/0020_fix_admin_blank.py b/arkindex/ponos/migrations/0020_fix_admin_blank.py
deleted file mode 100644
index d1e47bf09aec75e20c02f390d845067d087e9182..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0020_fix_admin_blank.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# Generated by Django 3.1.2 on 2020-10-19 10:58
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0019_secret"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="task",
- name="command",
- field=models.TextField(blank=True, null=True),
- ),
- migrations.AlterField(
- model_name="task",
- name="env",
- field=models.JSONField(blank=True, null=True),
- ),
- migrations.AlterField(
- model_name="task",
- name="image_artifact",
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="tasks_using_image",
- to="ponos.artifact",
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0021_add_ping_and_ram_fields.py b/arkindex/ponos/migrations/0021_add_ping_and_ram_fields.py
deleted file mode 100644
index d681f6af656ad549b669e85ba252f8966a3eb66a..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0021_add_ping_and_ram_fields.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Generated by Django 3.1 on 2020-10-27 10:44
-
-import django.core.validators
-from django.db import migrations, models
-
-
-def set_default_ping(apps, schema_editor):
- Agent = apps.get_model("ponos", "Agent")
- Agent.objects.all().update(last_ping=models.F("updated"))
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0020_fix_admin_blank"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="agent",
- name="cpu_load",
- field=models.FloatField(blank=True, null=True),
- ),
- migrations.AddField(
- model_name="agent",
- name="ram_load",
- field=models.FloatField(blank=True, null=True),
- ),
- migrations.AddField(
- model_name="agent",
- name="ram_total",
- field=models.BigIntegerField(
- default=1, validators=[django.core.validators.MinValueValidator(1)]
- ),
- preserve_default=False,
- ),
- migrations.AddField(
- model_name="agent",
- name="last_ping",
- field=models.DateTimeField(auto_now=True),
- preserve_default=False,
- ),
- migrations.RunPython(
- set_default_ping,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AlterField(
- model_name="agent",
- name="last_ping",
- field=models.DateTimeField(editable=False),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0022_rm_excluded_included_tags.py b/arkindex/ponos/migrations/0022_rm_excluded_included_tags.py
deleted file mode 100644
index ae2e173b008a6d203722a7a033a420d1205e85a5..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0022_rm_excluded_included_tags.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 3.1 on 2020-11-02 10:37
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0021_add_ping_and_ram_fields"),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name="agent",
- name="exclude_tags",
- ),
- migrations.RemoveField(
- model_name="agent",
- name="include_tags",
- ),
- ]
diff --git a/arkindex/ponos/migrations/0023_gpus.py b/arkindex/ponos/migrations/0023_gpus.py
deleted file mode 100644
index 1d9cf6b56357a8e8137cce4349519df7f1466b7b..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0023_gpus.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Generated by Django 3.1.2 on 2020-11-18 09:30
-
-import django.core.validators
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0022_rm_excluded_included_tags"),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name="agent",
- name="gpu_count",
- ),
- migrations.RemoveField(
- model_name="agent",
- name="gpu_names",
- ),
- migrations.CreateModel(
- name="GPU",
- fields=[
- (
- "id",
- models.UUIDField(primary_key=True, serialize=False),
- ),
- ("name", models.CharField(max_length=250)),
- ("index", models.PositiveIntegerField()),
- (
- "ram_total",
- models.BigIntegerField(
- validators=[django.core.validators.MinValueValidator(1)]
- ),
- ),
- (
- "agent",
- models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name="gpus",
- to="ponos.agent",
- ),
- ),
- ],
- options={
- "unique_together": {("agent_id", "index")},
- },
- ),
- migrations.AddField(
- model_name="task",
- name="gpu",
- field=models.OneToOneField(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="task",
- to="ponos.gpu",
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0024_task_requires_gpu.py b/arkindex/ponos/migrations/0024_task_requires_gpu.py
deleted file mode 100644
index 0d919b167e18a0e673c10073e1431d01223b302c..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0024_task_requires_gpu.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.2 on 2020-12-10 16:00
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0023_gpus"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="requires_gpu",
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0025_workflow_farm.py b/arkindex/ponos/migrations/0025_workflow_farm.py
deleted file mode 100644
index b679ae75622d79f82dada0c8b072ff7368a35b86..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0025_workflow_farm.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Generated by Django 3.1.5 on 2021-02-23 15:52
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-DEFAULT_FARM_ID = uuid.uuid4()
-
-
-def set_default_farm(apps, schema_editor):
- Workflow = apps.get_model("ponos", "Workflow")
- Farm = apps.get_model("ponos", "Farm")
- workflows = Workflow.objects.all()
- if not workflows.exists():
- return
- default_farm = Farm.objects.order_by().first()
- if not default_farm:
- # Create a default farm if required
- default_farm = Farm.objects.create(name="Default farm")
- workflows.update(farm_id=default_farm.id)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0024_task_requires_gpu"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="workflow",
- name="farm",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.PROTECT, null=True, to="ponos.farm"
- ),
- ),
- migrations.RunPython(
- set_default_farm,
- reverse_code=migrations.RunPython.noop,
- # No workflow exists initially
- elidable=True,
- ),
- migrations.AlterField(
- model_name="workflow",
- name="farm",
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.PROTECT,
- related_name="workflows",
- to="ponos.farm",
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0026_alter_artifact_size.py b/arkindex/ponos/migrations/0026_alter_artifact_size.py
deleted file mode 100644
index f2898ad0571057f98d4dddc92a07b117cecb5cf0..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0026_alter_artifact_size.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-07 09:09
-
-import django.core.validators
-from django.db import migrations, models
-
-
-def delete_large_artifacts(apps, schema_editor):
- Artifact = apps.get_model("ponos", "Artifact")
- Artifact.objects.filter(
- models.Q(size__lt=1) | models.Q(size__gt=5368709120)
- ).delete()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0025_workflow_farm"),
- ]
-
- operations = [
- migrations.RunPython(
- delete_large_artifacts, reverse_code=migrations.RunPython.noop
- ),
- migrations.AlterField(
- model_name="artifact",
- name="size",
- field=models.BigIntegerField(
- validators=[
- django.core.validators.MinValueValidator(1),
- django.core.validators.MaxValueValidator(5368709120),
- ]
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0027_task_priority.py b/arkindex/ponos/migrations/0027_task_priority.py
deleted file mode 100644
index cb1033c6a0f7624de65e1f21fb0d89a3353cb5d9..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0027_task_priority.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.5 on 2021-07-20 10:56
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0026_alter_artifact_size"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="priority",
- field=models.PositiveIntegerField(default=10),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0028_workflow_finished.py b/arkindex/ponos/migrations/0028_workflow_finished.py
deleted file mode 100644
index fef4a65faef0718ae7a65f5fbf5fe331305553bf..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0028_workflow_finished.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 4.0.1 on 2022-02-15 09:55
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0027_task_priority"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="workflow",
- name="finished",
- field=models.DateTimeField(blank=True, null=True),
- ),
- migrations.AddConstraint(
- model_name="workflow",
- constraint=models.CheckConstraint(
- check=models.Q(finished=None)
- | models.Q(finished__gte=models.F("created")),
- name="ponos_workflow_finished_after_created",
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0029_task_expiry.py b/arkindex/ponos/migrations/0029_task_expiry.py
deleted file mode 100644
index 662f5f7e32aae84574116f97f19064f9af2f6908..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0029_task_expiry.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Generated by Django 4.0.1 on 2022-03-28 15:53
-
-from datetime import timedelta
-
-from django.db import migrations, models
-
-from arkindex.ponos.models import expiry_default
-
-
-def set_expiry(apps, schema_editor):
- Task = apps.get_model("ponos", "Task")
- Task.objects.update(expiry=models.F("updated") + timedelta(days=30))
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0028_workflow_finished"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="expiry",
- field=models.DateTimeField(null=True),
- ),
- migrations.RunPython(
- set_expiry,
- reverse_code=migrations.RunPython.noop,
- ),
- migrations.AlterField(
- model_name="task",
- name="expiry",
- field=models.DateTimeField(default=expiry_default),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0030_task_extra_files.py b/arkindex/ponos/migrations/0030_task_extra_files.py
deleted file mode 100644
index f36fa9f7c7526f369725a2e7f06183388e53aa5d..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0030_task_extra_files.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.0.2 on 2022-06-07 11:33
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0029_task_expiry"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="extra_files",
- field=models.JSONField(default=dict),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0031_emptyable_jsonfield.py b/arkindex/ponos/migrations/0031_emptyable_jsonfield.py
deleted file mode 100644
index c18fd739acc0b913bc1286f773ed2189f6d5608b..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0031_emptyable_jsonfield.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 4.0.5 on 2022-06-27 15:03
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0030_task_extra_files"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="task",
- name="env",
- field=models.JSONField(blank=True, null=True),
- ),
- migrations.AlterField(
- model_name="task",
- name="extra_files",
- field=models.JSONField(default=dict),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0032_stringify_json.py b/arkindex/ponos/migrations/0032_stringify_json.py
deleted file mode 100644
index ae2a768810615fb3084ca7c1a59be5400a5b7ed4..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0032_stringify_json.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Generated by Django 4.0.5 on 2022-06-27 15:08
-
-from django.db import migrations
-
-
-def stringify_json(apps, schema_editor):
- if schema_editor.connection.vendor == "postgresql":
- # Postgres' jsonb type allows to stringify all payloads in one query per column
- schema_editor.execute(
- """
- UPDATE ponos_task
- SET env = (
- SELECT jsonb_object(array_agg(key), array_agg(value))
- FROM jsonb_each_text(env)
- )
- WHERE id in (
- SELECT id FROM ponos_task, jsonb_each(env) AS env_items
- WHERE jsonb_typeof(env_items.value) <> 'string'
- );
- """
- )
- schema_editor.execute(
- """
- UPDATE ponos_task
- SET extra_files = (
- SELECT jsonb_object(array_agg(key), array_agg(value))
- FROM jsonb_each_text(extra_files)
- )
- WHERE id in (
- SELECT id FROM ponos_task, jsonb_each(extra_files) AS extra_files_items
- WHERE jsonb_typeof(extra_files_items.value) <> 'string'
- );
- """
- )
-
- else:
- Task = apps.get_model("ponos", "Task")
- to_update = []
- for task in Task.objects.only("id", "env", "extra_files"):
- updated = False
-
- if task.env and not all(isinstance(value, str) for value in task.env):
- task.env = {key: str(value) for key, value in task.env.items()}
- updated = True
-
- if task.extra_files and not all(
- isinstance(value, str) for value in task.extra_files
- ):
- task.extra_files = {
- key: str(value) for key, value in task.extra_files.items()
- }
- updated = True
-
- if updated:
- to_update.append(task)
-
- Task.objects.bulk_update(to_update, fields=["env", "extra_files"])
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0031_emptyable_jsonfield"),
- ]
-
- operations = [
- migrations.RunPython(stringify_json, reverse_code=migrations.RunPython.noop),
- ]
diff --git a/arkindex/ponos/migrations/0033_task_shm_size.py b/arkindex/ponos/migrations/0033_task_shm_size.py
deleted file mode 100644
index 9623c8b669b387c17df7a2bc4a074b8973bd495d..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0033_task_shm_size.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 4.0.4 on 2022-10-20 14:37
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0032_stringify_json"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="task",
- name="shm_size",
- field=models.CharField(
- blank=True, editable=False, max_length=80, null=True
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0034_alter_artifact_size.py b/arkindex/ponos/migrations/0034_alter_artifact_size.py
deleted file mode 100644
index 94a378d25a725b5bbde2bd66f41f25660898147f..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0034_alter_artifact_size.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.1.3 on 2023-01-09 09:52
-
-from django.core.validators import MinValueValidator
-from django.db import migrations, models
-
-from arkindex.ponos.models import artifact_max_size
-from arkindex.ponos.validators import MaxValueValidator
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("ponos", "0033_task_shm_size"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="artifact",
- name="size",
- field=models.BigIntegerField(
- validators=[MinValueValidator(1), MaxValueValidator(artifact_max_size)]
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0035_alter_task_tags.py b/arkindex/ponos/migrations/0035_alter_task_tags.py
deleted file mode 100644
index 82e05b839acc74c1a9ebc0988c55a73a358c439d..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0035_alter_task_tags.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from django.db import migrations, models
-
-import arkindex.project.fields
-
-
-def comma_sep_char_to_list(apps, schema_editor):
- Task = apps.get_model("ponos", "Task")
- tasks_attrs = Task.objects.using('default').values_list("id", "old_tags")
- Task.objects.bulk_update(
- (
- Task(
- id=task_id,
- tags=old_tags.split(",") if old_tags else []
- ) for task_id, old_tags in tasks_attrs
- ),
- ['tags']
- )
-
-
-def revert_comma_sep_char_to_list(apps, schema_editor):
- Task = apps.get_model("ponos", "Task")
- tasks_attrs = Task.objects.using('default').values_list("id", "tags")
- Task.objects.bulk_update(
- (
- Task(
- id=task_id,
- old_tags=",".join(tags)
- ) for task_id, tags in tasks_attrs
- ),
- ['old_tags']
- )
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0034_alter_artifact_size'),
- ]
-
- operations = [
- migrations.RenameField(
- model_name="task",
- old_name="tags",
- new_name="old_tags",
- ),
- migrations.AddField(
- model_name='task',
- name='tags',
- field=arkindex.project.fields.ArrayField(
- base_field=models.CharField(max_length=250),
- blank=True,
- default=list,
- size=16,
- ),
- ),
- migrations.RunPython(
- comma_sep_char_to_list,
- reverse_code=revert_comma_sep_char_to_list,
- ),
- migrations.RemoveField(
- model_name='task',
- name='old_tags',
- ),
- ]
diff --git a/arkindex/ponos/migrations/0036_hstore_task_env_and_extra_files.py b/arkindex/ponos/migrations/0036_hstore_task_env_and_extra_files.py
deleted file mode 100644
index b2a1c4270b0f02f440298ee36d6ff5d56584acfd..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0036_hstore_task_env_and_extra_files.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Generated by Django 4.1.5 on 2023-02-03 08:39
-
-import django.contrib.postgres.fields.hstore
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0035_alter_task_tags'),
- ]
-
- operations = [
- migrations.RenameField(
- model_name="task",
- old_name="env",
- new_name="old_env",
- ),
- migrations.RenameField(
- model_name="task",
- old_name="extra_files",
- new_name="old_extra_files",
- ),
- migrations.AddField(
- model_name='task',
- name='env',
- field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
- ),
- migrations.AddField(
- model_name='task',
- name='extra_files',
- field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
- ),
- migrations.RunSQL(
- """
- UPDATE ponos_task SET
- env = (
- SELECT COALESCE(hstore(array_agg(key), array_agg(value)), hstore(''))
- FROM jsonb_each_text(old_env)
- ),
- extra_files = (
- SELECT COALESCE(hstore(array_agg(key), array_agg(value)), hstore(''))
- FROM jsonb_each_text(old_extra_files)
- )
- ;
- """,
- reverse_sql="""
- UPDATE ponos_task SET
- old_env = env::jsonb,
- old_extra_files = extra_files::jsonb
- ;
- """
- ),
- migrations.RemoveField(
- model_name='task',
- name='old_env',
- ),
- migrations.RemoveField(
- model_name='task',
- name='old_extra_files',
- ),
- ]
diff --git a/arkindex/ponos/migrations/0037_task_token.py b/arkindex/ponos/migrations/0037_task_token.py
deleted file mode 100644
index bb11372fc4392f47d87393344bb5c11ec4611a23..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0037_task_token.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# Generated by Django 4.1.7 on 2023-03-07 15:19
-
-from django.db import migrations, models
-
-from arkindex.ponos.models import task_token_default
-
-
-def add_task_tokens(apps, schema_editor):
- Task = apps.get_model('ponos', 'Task')
- to_update = []
- for task in Task.objects.filter(token=None).only('id').iterator():
- task.token = task_token_default()
- to_update.append(task)
- Task.objects.bulk_update(to_update, ['token'], batch_size=100)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0036_hstore_task_env_and_extra_files'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='task',
- name='token',
- field=models.CharField(
- max_length=52,
- # Make the field temporarily nullable and not unique, so that we can
- # fill the tokens on existing tasks before adding the constraints.
- null=True,
- ),
- ),
- migrations.RunPython(
- add_task_tokens,
- reverse_code=migrations.RunPython.noop,
- ),
- ]
diff --git a/arkindex/ponos/migrations/0038_task_token_unique.py b/arkindex/ponos/migrations/0038_task_token_unique.py
deleted file mode 100644
index 5de95c85ee7afba67531028d0fd1fb9eaac1c586..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0038_task_token_unique.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.1.7 on 2023-03-07 15:25
-
-from django.db import migrations, models
-
-from arkindex.ponos.models import task_token_default
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0037_task_token'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='task',
- name='token',
- field=models.CharField(
- default=task_token_default,
- max_length=52,
- unique=True,
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0039_keep_task_gpu.py b/arkindex/ponos/migrations/0039_keep_task_gpu.py
deleted file mode 100644
index 9e32b31ec8583cd7de6271fca54114570279280d..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0039_keep_task_gpu.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 4.1.7 on 2023-04-20 08:06
-
-from django.db import migrations, models
-
-from arkindex.ponos.models import ACTIVE_STATES
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0038_task_token_unique'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='task',
- name='gpu',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=models.SET_NULL,
- related_name='tasks',
- to='ponos.gpu',
- ),
- ),
- migrations.AddConstraint(
- model_name='task',
- constraint=models.UniqueConstraint(
- 'gpu',
- condition=models.Q(state__in=ACTIVE_STATES),
- name='unique_gpu_on_active_tasks',
- ),
- ),
- ]
diff --git a/arkindex/ponos/migrations/0040_remove_workflow_recipe.py b/arkindex/ponos/migrations/0040_remove_workflow_recipe.py
deleted file mode 100644
index 12a62ed997fde3af910006a169aaaa4afff21bfb..0000000000000000000000000000000000000000
--- a/arkindex/ponos/migrations/0040_remove_workflow_recipe.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.1.7 on 2023-04-28 13:07
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0039_keep_task_gpu'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='workflow',
- name='recipe',
- ),
- ]
diff --git a/arkindex/process/migrations/0001_initial.py b/arkindex/process/migrations/0001_initial.py
index 058bda1a3cc5da7e1ea2c562bb667c2997e5bbd3..43fbdce9a47026fd4593729ce0504bf8f46c564e 100644
--- a/arkindex/process/migrations/0001_initial.py
+++ b/arkindex/process/migrations/0001_initial.py
@@ -1,36 +1,42 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import uuid
-import django.contrib.postgres.fields.jsonb
+import django.core.validators
+import django.db.models.deletion
import enumfields.fields
from django.db import migrations, models
-from enumfields import Enum
import arkindex.process.models
import arkindex.project.aws
import arkindex.project.fields
-class EventType(Enum):
- Addition = 'A'
- Edit = 'M'
- Deletion = 'D'
-
class Migration(migrations.Migration):
initial = True
+ dependencies = [
+ ('ponos', '0001_initial'),
+ ('documents', '0001_initial'),
+ ]
+
operations = [
+ migrations.CreateModel(
+ name='CorpusWorkerVersion',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ],
+ ),
migrations.CreateModel(
name='DataFile',
fields=[
- ('hash', arkindex.project.fields.MD5HashField(max_length=32)),
- ('status', enumfields.fields.EnumField(default='unchecked', enum=arkindex.project.aws.S3FileStatus, max_length=50)),
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=100)),
+ ('name', models.CharField(max_length=255)),
('size', models.PositiveIntegerField(help_text='file size in bytes')),
- ('content_type', models.CharField(max_length=50)),
+ ('content_type', models.CharField(max_length=120)),
+ ('status', enumfields.fields.EnumField(default='unchecked', enum=arkindex.project.aws.S3FileStatus, max_length=50)),
+ ('trashed', models.BooleanField(default=False)),
],
options={
'ordering': ['corpus', 'name'],
@@ -38,29 +44,40 @@ class Migration(migrations.Migration):
bases=(arkindex.project.aws.S3FileMixin, models.Model),
),
migrations.CreateModel(
- name='DataImport',
+ name='GitRef',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('type', enumfields.fields.EnumField(enum=arkindex.process.models.GitRefType, max_length=10)),
+ ('name', models.CharField(max_length=250)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Process',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
+ ('name', models.CharField(blank=True, max_length=100, null=True)),
('mode', enumfields.fields.EnumField(enum=arkindex.process.models.ProcessMode, max_length=30)),
- ('payload', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
+ ('activity_state', enumfields.fields.EnumField(default='disabled', enum=arkindex.process.models.ActivityState, max_length=32)),
+ ('name_contains', models.CharField(blank=True, max_length=250, null=True)),
+ ('load_children', models.BooleanField(default=False)),
+ ('collection_id', models.PositiveIntegerField(blank=True, null=True)),
+ ('use_cache', models.BooleanField(default=False)),
+ ('use_gpu', models.BooleanField(blank=True, default=False)),
+ ('bucket_name', models.CharField(blank=True, max_length=63, null=True, validators=[django.core.validators.MinLengthValidator(3)])),
+ ('prefix', models.CharField(blank=True, max_length=1024, null=True)),
],
options={
+ 'verbose_name_plural': 'processes',
'ordering': ['corpus', '-created'],
},
),
migrations.CreateModel(
- name='Event',
+ name='ProcessElement',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('created', models.DateTimeField(auto_now_add=True)),
- ('updated', models.DateTimeField(auto_now=True)),
- ('type', enumfields.fields.EnumField(enum=EventType, max_length=10)),
],
- options={
- 'ordering': ['element_id', '-created'],
- },
),
migrations.CreateModel(
name='Repository',
@@ -68,7 +85,6 @@ class Migration(migrations.Migration):
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('url', models.URLField(unique=True)),
('hook_token', models.CharField(max_length=250, unique=True)),
- ('provider_name', models.CharField(choices=[('GitLabProvider', 'GitLab')], max_length=50)),
],
options={
'verbose_name_plural': 'repositories',
@@ -81,9 +97,76 @@ class Migration(migrations.Migration):
('updated', models.DateTimeField(auto_now=True)),
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('hash', models.CharField(max_length=50)),
- ('ref', models.CharField(max_length=50)),
('message', models.TextField()),
('author', models.CharField(max_length=50)),
],
),
+ migrations.CreateModel(
+ name='Worker',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=100)),
+ ('slug', models.CharField(max_length=100)),
+ ('public', models.BooleanField(default=False)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='WorkerActivity',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now_add=True)),
+ ('started', models.DateTimeField(blank=True, null=True)),
+ ('state', enumfields.fields.EnumField(default='queued', enum=arkindex.process.models.WorkerActivityState, max_length=10)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='WorkerConfiguration',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('name', models.CharField(max_length=250)),
+ ('configuration', models.JSONField(default=dict)),
+ ('configuration_hash', arkindex.project.fields.MD5HashField(max_length=32)),
+ ('archived', models.BooleanField(default=False)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='WorkerType',
+ fields=[
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('slug', models.SlugField(max_length=100, unique=True)),
+ ('display_name', models.CharField(max_length=100)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
+ migrations.CreateModel(
+ name='WorkerVersion',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('configuration', models.JSONField()),
+ ('state', enumfields.fields.EnumField(default='created', enum=arkindex.process.models.WorkerVersionState, max_length=10)),
+ ('gpu_usage', enumfields.fields.EnumField(blank=True, default='disabled', enum=arkindex.process.models.WorkerVersionGPUUsage, max_length=10)),
+ ('model_usage', models.BooleanField(default=False)),
+ ('docker_image_iid', models.CharField(blank=True, max_length=80, null=True)),
+ ('corpora', models.ManyToManyField(related_name='worker_versions', through='process.CorpusWorkerVersion', to='documents.corpus')),
+ ('docker_image', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ponos.artifact')),
+ ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='process.revision')),
+ ('worker', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='process.worker')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='WorkerRun',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('parents', arkindex.project.fields.ArrayField(base_field=models.UUIDField(), size=None)),
+ ('summary', models.TextField()),
+ ('configuration', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='worker_runs', to='process.workerconfiguration')),
+ ],
+ ),
]
diff --git a/arkindex/process/migrations/0002_initial.py b/arkindex/process/migrations/0002_initial.py
index b332e6ed4c1cbd09f25d4b93e446e93a244aa3f2..24acaea602cbdd6f5bc31bf445a6d5995010863d 100644
--- a/arkindex/process/migrations/0002_initial.py
+++ b/arkindex/process/migrations/0002_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import django.db.models.deletion
from django.db import migrations, models
@@ -9,36 +9,65 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
- ('documents', '0001_initial'),
+ ('training', '0001_initial'),
('process', '0001_initial'),
+ ('documents', '0002_initial'),
]
operations = [
migrations.AddField(
- model_name='revision',
- name='elements',
- field=models.ManyToManyField(
- related_name='revisions',
- through='process.Event',
- to='documents.Element',
- ),
+ model_name='workerrun',
+ name='model_version',
+ field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='worker_runs', to='training.modelversion'),
+ ),
+ migrations.AddField(
+ model_name='workerrun',
+ name='process',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='worker_runs', to='process.process'),
+ ),
+ migrations.AddField(
+ model_name='workerrun',
+ name='version',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='worker_runs', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='workerconfiguration',
+ name='worker',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='configurations', to='process.worker'),
+ ),
+ migrations.AddField(
+ model_name='workeractivity',
+ name='configuration',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='worker_activities', to='process.workerconfiguration'),
+ ),
+ migrations.AddField(
+ model_name='workeractivity',
+ name='element',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activities', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='workeractivity',
+ name='process',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activities', to='process.process'),
+ ),
+ migrations.AddField(
+ model_name='workeractivity',
+ name='worker_version',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activities', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='worker',
+ name='repository',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='workers', to='process.repository'),
+ ),
+ migrations.AddField(
+ model_name='worker',
+ name='type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='type', to='process.workertype'),
),
migrations.AddField(
model_name='revision',
name='repo',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='revisions',
- to='process.Repository',
- ),
- ),
- migrations.AddField(
- model_name='repository',
- name='corpus',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='repos',
- to='documents.Corpus',
- ),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='process.repository'),
),
]
diff --git a/arkindex/process/migrations/0003_initial.py b/arkindex/process/migrations/0003_initial.py
index 16fbfe3480a8db4696975373dd7066192de90311..18b9f8989b6de308a75c9707c92d41f50ba6585b 100644
--- a/arkindex/process/migrations/0003_initial.py
+++ b/arkindex/process/migrations/0003_initial.py
@@ -1,120 +1,218 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
+import arkindex.process.models
+import pgtrigger.compiler
+import pgtrigger.migrations
+
class Migration(migrations.Migration):
initial = True
dependencies = [
+ ('training', '0001_initial'),
+ ('ponos', '0001_initial'),
+ ('users', '0001_initial'),
('process', '0002_initial'),
+ ('documents', '0003_initial'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('documents', '0002_initial'),
- ('ponos', '0012_advanced_tags'),
- ('users', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='repository',
name='credentials',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='repos',
- to='users.OAuthCredentials',
- ),
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='repos', to='users.oauthcredentials'),
+ ),
+ migrations.AddField(
+ model_name='repository',
+ name='git_ref_revisions',
+ field=models.ManyToManyField(through='process.GitRef', to='process.revision'),
),
migrations.AddField(
- model_name='event',
+ model_name='processelement',
name='element',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='events',
- to='documents.Element',
- ),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='process_elements', to='documents.element'),
),
migrations.AddField(
- model_name='event',
- name='revision',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name='events',
- to='process.Revision',
- ),
+ model_name='processelement',
+ name='process',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='process_elements', to='process.process'),
),
migrations.AddField(
- model_name='dataimport',
+ model_name='process',
name='corpus',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='imports',
- to='documents.Corpus',
- ),
+ field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='documents.corpus'),
),
migrations.AddField(
- model_name='dataimport',
+ model_name='process',
name='creator',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='imports',
- to=settings.AUTH_USER_MODEL,
- ),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='processes', to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
- model_name='dataimport',
+ model_name='process',
+ name='element',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='element_type',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='element_imports', to='documents.elementtype'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='elements',
+ field=models.ManyToManyField(related_name='processes', through='process.ProcessElement', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='process',
name='files',
- field=models.ManyToManyField(
- related_name='imports',
- to='process.DataFile',
- ),
+ field=models.ManyToManyField(related_name='processes', to='process.datafile'),
),
migrations.AddField(
- model_name='dataimport',
+ model_name='process',
+ name='folder_type',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='folder_imports', to='documents.elementtype'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='model',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='training_processes', to='training.model'),
+ ),
+ migrations.AddField(
+ model_name='process',
name='revision',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name='dataimports',
- to='process.Revision',
- ),
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='process.revision'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='template',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='applications', to='process.process'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='test_folder',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='test_folder_processes', to='documents.element'),
),
migrations.AddField(
- model_name='dataimport',
+ model_name='process',
+ name='train_folder',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='train_folder_processes', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='validation_folder',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='validation_folder_processes', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='process',
+ name='versions',
+ field=models.ManyToManyField(related_name='processes', through='process.WorkerRun', to='process.workerversion'),
+ ),
+ migrations.AddField(
+ model_name='process',
name='workflow',
- field=models.OneToOneField(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- to='ponos.Workflow',
- ),
+ field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ponos.workflow'),
+ ),
+ migrations.AddField(
+ model_name='gitref',
+ name='repository',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='refs', to='process.repository'),
+ ),
+ migrations.AddField(
+ model_name='gitref',
+ name='revision',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='refs', to='process.revision'),
),
migrations.AddField(
model_name='datafile',
name='corpus',
- field=models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='files',
- to='documents.Corpus',
- ),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='corpusworkerversion',
+ name='corpus',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='worker_version_cache', to='documents.corpus'),
+ ),
+ migrations.AddField(
+ model_name='corpusworkerversion',
+ name='worker_version',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='corpus_cache', to='process.workerversion'),
+ ),
+ migrations.AddConstraint(
+ model_name='workerversion',
+ constraint=models.CheckConstraint(check=models.Q(('docker_image_id', None), ('state', arkindex.process.models.WorkerVersionState['Available']), _negated=True), name='workerversion_available_requires_docker_image'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='workerversion',
+ unique_together={('worker', 'revision')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='workerrun',
+ unique_together={('version', 'process')},
+ ),
+ migrations.AddConstraint(
+ model_name='workerconfiguration',
+ constraint=models.CheckConstraint(check=models.Q(('configuration__typeof', 'object')), name='worker_configuration_configuration_objects'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='workerconfiguration',
+ unique_together={('worker', 'name'), ('worker', 'configuration_hash')},
+ ),
+ migrations.AddConstraint(
+ model_name='workeractivity',
+ constraint=models.UniqueConstraint(condition=models.Q(('configuration__isnull', True)), fields=('worker_version', 'element'), name='worker_activity_unique_no_configuration'),
+ ),
+ migrations.AddConstraint(
+ model_name='workeractivity',
+ constraint=models.UniqueConstraint(condition=models.Q(('configuration__isnull', False)), fields=('worker_version', 'element', 'configuration'), name='worker_activity_unique_configuration'),
+ ),
+ migrations.AddConstraint(
+ model_name='workeractivity',
+ constraint=models.CheckConstraint(check=models.Q(models.Q(('state', arkindex.process.models.WorkerActivityState['Started']), _negated=True), ('started__isnull', False), _connector='OR'), name='worker_activity_started_requires_started'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='worker',
+ unique_together={('slug', 'repository')},
),
migrations.AlterUniqueTogether(
name='revision',
unique_together={('repo', 'hash')},
),
migrations.AlterUniqueTogether(
- name='event',
- unique_together={('element', 'revision')},
+ name='processelement',
+ unique_together={('process', 'element')},
+ ),
+ migrations.AddConstraint(
+ model_name='process',
+ constraint=models.CheckConstraint(check=models.Q(models.Q(('mode', arkindex.process.models.ProcessMode['Local']), _negated=True), ('workflow', None), _connector='OR'), name='local_process_no_workflow', violation_error_message='Local processes cannot be started.'),
+ ),
+ migrations.AddConstraint(
+ model_name='process',
+ constraint=models.CheckConstraint(check=models.Q(('mode__in', (arkindex.process.models.ProcessMode['Local'], arkindex.process.models.ProcessMode['Repository'])), models.Q(('corpus', None), _negated=True), _connector='XOR'), name='check_process_corpus', violation_error_message='Local and repository processes cannot have a corpus, and other modes must have one set.'),
+ ),
+ migrations.AddConstraint(
+ model_name='process',
+ constraint=models.UniqueConstraint(models.F('creator'), condition=models.Q(('mode', arkindex.process.models.ProcessMode['Local'])), name='unique_local_process', violation_error_message='Only one local process is allowed per user.'),
),
migrations.AlterUniqueTogether(
- name='datafile',
- unique_together={('corpus', 'hash')},
+ name='gitref',
+ unique_together={('name', 'repository')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='corpusworkerversion',
+ unique_together={('corpus', 'worker_version')},
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name='workeractivity',
+ trigger=pgtrigger.compiler.Trigger(name='update_workeractivity_updated', sql=pgtrigger.compiler.UpsertTriggerSql(func='NEW.updated = now(); RETURN NEW;', hash='1e5a8fa0718f420e6cd4f2a31434cd39a9c9bc67', operation='UPDATE', pgid='pgtrigger_update_workeractivity_updated_f2812', table='process_workeractivity', when='BEFORE')),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name='workeractivity',
+ trigger=pgtrigger.compiler.Trigger(name='read_only_workeractivity_updated', sql=pgtrigger.compiler.UpsertTriggerSql(condition='WHEN (OLD."updated" IS DISTINCT FROM (NEW."updated"))', func="RAISE EXCEPTION 'pgtrigger: Cannot update rows from % table', TG_TABLE_NAME;", hash='6276c6971a1d2669659e407418e2db1fa7dc6965', operation='UPDATE', pgid='pgtrigger_read_only_workeractivity_updated_a80ab', table='process_workeractivity', when='BEFORE')),
),
]
diff --git a/arkindex/process/migrations/0004_remove_event.py b/arkindex/process/migrations/0004_remove_event.py
deleted file mode 100644
index 913ddf1ad967c66c1b04a861b3200570c35285e0..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0004_remove_event.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 2.2.11 on 2020-04-09 12:10
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0003_initial'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='revision',
- name='elements',
- ),
- migrations.DeleteModel(
- name='Event',
- ),
- ]
diff --git a/arkindex/process/migrations/0005_filename_max_length.py b/arkindex/process/migrations/0005_filename_max_length.py
deleted file mode 100644
index 837054dd10ab89565a3fb21dae16f8dd5a0c396a..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0005_filename_max_length.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.2.10 on 2020-04-21 09:04
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0004_remove_event'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='datafile',
- name='name',
- field=models.CharField(max_length=255),
- ),
- ]
diff --git a/arkindex/process/migrations/0006_revision_state.py b/arkindex/process/migrations/0006_revision_state.py
deleted file mode 100644
index 4db90b40e4c4009427de52de751e01b6dae0d69f..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0006_revision_state.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Generated by Django 2.2.11 on 2020-04-30 11:00
-
-import enumfields.fields
-from django.db import migrations
-
-
-class RevisionState(enumfields.fields.Enum):
- Created = 'created'
- Processing = 'processing'
- Available = 'available'
- Error = 'error'
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0005_filename_max_length'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='revision',
- name='state',
- field=enumfields.fields.EnumField(
- default='created',
- enum=RevisionState,
- max_length=10
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0007_worker_workerversion.py b/arkindex/process/migrations/0007_worker_workerversion.py
deleted file mode 100644
index 98179e9e1a05facb76d6e8805fd26c12025016ce..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0007_worker_workerversion.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Generated by Django 2.2.11 on 2020-05-19 14:55
-
-import uuid
-from enum import Enum
-
-import django.contrib.postgres.fields.jsonb
-import django.db.models.deletion
-import enumfields.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0006_revision_state'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Worker',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=100)),
- ('slug', models.CharField(max_length=100)),
- ('type', enumfields.fields.EnumField(enum=Enum('MLToolType', ''), max_length=50)),
- ('repository', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='workers',
- to='process.Repository'
- )),
- ],
- options={
- 'unique_together': {('slug', 'repository')},
- },
- ),
- migrations.CreateModel(
- name='WorkerVersion',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('configuration', django.contrib.postgres.fields.jsonb.JSONField()),
- ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='process.Revision')),
- ('worker', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='process.Worker')),
- ],
- options={
- 'unique_together': {('worker', 'revision')},
- },
- ),
- ]
diff --git a/arkindex/process/migrations/0008_add_gitref.py b/arkindex/process/migrations/0008_add_gitref.py
deleted file mode 100644
index 80344263d176f92ab5c80ff286eb559731b424a9..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0008_add_gitref.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Generated by Django 2.2.11 on 2020-05-21 15:34
-
-import uuid
-
-import django.db.models.deletion
-import enumfields.fields
-from django.db import migrations, models
-
-import arkindex.process.models
-
-
-def migrate_git_refs(apps, schema_editor):
- """
- Used to convert values on the ref attribute from Revision to a GitRef object.
- The ref attribute on Revision only supported branch-type references, so no need to
- worry about the type of the reference.
-
- Some references are duplicated on multiple revisions (e.g., two master branches), in
- which case the GitRef object is linked to the latest version returned by the default
- DB order. These values will be updated later when the GitLab webhook will be triggered.
- """
- db_alias = schema_editor.connection.alias
- Revision = apps.get_model('process', 'Revision')
- for rev in Revision.objects.using(db_alias).filter(ref__isnull=False):
- ref = rev.repo.refs.filter(name=rev.ref).first()
- if ref:
- ref.revision = rev
- ref.save()
- else:
- rev.refs.create(
- name=rev.ref,
- type=arkindex.process.models.GitRefType('branch'),
- repository=rev.repo
- )
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0007_worker_workerversion'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='GitRef',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('type', enumfields.fields.EnumField(enum=arkindex.process.models.GitRefType, max_length=10)),
- ('name', models.CharField(max_length=250)),
- ('repository', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='refs',
- to='process.Repository'
- )),
- ('revision', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='refs',
- to='process.Revision'
- )),
- ],
- options={
- 'unique_together': {('name', 'repository')},
- },
- ),
- migrations.RunPython(
- migrate_git_refs,
- reverse_code=migrations.RunPython.noop,
- ),
- migrations.AddField(
- model_name='repository',
- name='git_ref_revisions',
- field=models.ManyToManyField(through='process.GitRef', to='process.Revision'),
- ),
- migrations.RemoveField(
- model_name='revision',
- name='ref',
- ),
- ]
diff --git a/arkindex/process/migrations/0009_remove_datafile_hash.py b/arkindex/process/migrations/0009_remove_datafile_hash.py
deleted file mode 100644
index 27f06b83897ba2f17b8612aa895df543b251e6ad..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0009_remove_datafile_hash.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-08 10:25
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0008_add_gitref'),
- ]
-
- operations = [
- migrations.AlterUniqueTogether(
- name='datafile',
- unique_together=set(),
- ),
- migrations.RemoveField(
- model_name='datafile',
- name='hash',
- ),
- ]
diff --git a/arkindex/process/migrations/0010_workerversion_docker_image.py b/arkindex/process/migrations/0010_workerversion_docker_image.py
deleted file mode 100644
index 6c17f7b90cf185957ac11145eaf7527ed45cb57d..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0010_workerversion_docker_image.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-18 12:52
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('ponos', '0015_task_has_docker_socket'),
- ('process', '0009_remove_datafile_hash'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerversion',
- name='docker_image',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='ponos.Artifact'),
- ),
- ]
diff --git a/arkindex/process/migrations/0011_refactor_payload_dataimport.py b/arkindex/process/migrations/0011_refactor_payload_dataimport.py
deleted file mode 100644
index 997a9075d5ed6e3a422b540f4a019c0032a26603..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0011_refactor_payload_dataimport.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-24 09:39
-
-import django.db.models.deletion
-from django.db import migrations, models
-from enumfields import Enum
-
-
-class OldDataImportMode(Enum):
- Elements = 'elements'
-
-
-def populate_new_fields(apps, schema_editor):
- """
- Migrate all elements selection fields from a DataImport.payload towards new DB fields on this model
- so they can be edited later on.
- It also means that a DataImport can now be configured without being started immediately.
- """
- db_alias = schema_editor.connection.alias
- DataImport = apps.get_model('process', 'DataImport')
- ElementType = apps.get_model('documents', 'ElementType')
- Element = apps.get_model('documents', 'Element')
-
- # Switch to Enum to restore the removed Elements import mode to allow filtering on it
- DataImport.mode.field.enum = OldDataImportMode
-
- for di in DataImport.objects.using(db_alias).filter(mode=OldDataImportMode.Elements):
- # There, we retrieve the payload elements that interest us to populate the new fields. We also remove the
- # attributes "ml_tools", "thumbnails" and "chunks" from the payload because they will no longer be useful.
- element, name_contains, elt_type, selection, best_class, _, _, _ = map(
- lambda key: di.payload.pop(key, None),
- (
- 'element',
- 'name_contains',
- 'element_type',
- 'elements',
- 'best_class',
- 'ml_tools',
- 'thumbnails',
- 'chunks'
- )
- )
-
- if name_contains:
- di.name_contains = name_contains
- if elt_type:
- try:
- di.element_type = ElementType.objects.using(db_alias).get(corpus=di.corpus, slug=elt_type)
- except ElementType.DoesNotExist:
- pass
- if element:
- try:
- di.element = Element.objects.using(db_alias).get(id=element)
- except Element.DoesNotExist:
- pass
- if selection:
- di.use_selection = True
- if best_class:
- di.best_class = best_class
-
- # At the end of the migration, we will find, in the payload, only elements that are not related to an Elements DataImport.
- di.save()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0015_elementtype_allowed_transcription'),
- ('process', '0010_workerversion_docker_image'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='best_class',
- field=models.CharField(blank=True, max_length=150, null=True),
- ),
- migrations.AddField(
- model_name='dataimport',
- name='element',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.Element'),
- ),
- migrations.AddField(
- model_name='dataimport',
- name='element_type',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.ElementType'),
- ),
- migrations.AddField(
- model_name='dataimport',
- name='name_contains',
- field=models.CharField(blank=True, max_length=150, null=True),
- ),
- migrations.AddField(
- model_name='dataimport',
- name='use_selection',
- field=models.BooleanField(default=False),
- ),
- migrations.RunPython(
- populate_new_fields,
- reverse_code=migrations.RunPython.noop,
- ),
- ]
diff --git a/arkindex/process/migrations/0012_move_revision_state.py b/arkindex/process/migrations/0012_move_revision_state.py
deleted file mode 100644
index 2962c17b1ff17fd6a5c59b0b2fb44a03aa68ad6c..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0012_move_revision_state.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-25 09:40
-
-import django.db.models.deletion
-import enumfields.fields
-from django.db import migrations, models
-
-import arkindex.process.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0011_refactor_payload_dataimport'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='revision',
- name='state',
- ),
- migrations.AddField(
- model_name='workerversion',
- name='state',
- field=enumfields.fields.EnumField(default='created', enum=arkindex.process.models.WorkerVersionState, max_length=10),
- ),
- migrations.AlterField(
- model_name='workerversion',
- name='revision',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='process.Revision'),
- ),
- ]
diff --git a/arkindex/process/migrations/0013_create_model_workerrun.py b/arkindex/process/migrations/0013_create_model_workerrun.py
deleted file mode 100644
index 696e558ad8bafa9c9fd4a3ce01b140bcc3013578..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0013_create_model_workerrun.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-29 13:18
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-import arkindex.project.fields
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0012_move_revision_state'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='WorkerRun',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('parents', arkindex.project.fields.ArrayField(base_field=models.UUIDField(), size=None)),
- ('dataimport', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='worker_runs', to='process.DataImport')),
- ('version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='worker_runs', to='process.WorkerVersion')),
- ],
- options={
- 'unique_together': {('version', 'dataimport')},
- },
- ),
- migrations.AddField(
- model_name='dataimport',
- name='versions',
- field=models.ManyToManyField(related_name='imports', through='process.WorkerRun', to='process.WorkerVersion'),
- ),
- ]
diff --git a/arkindex/process/migrations/0014_dataimport_selection.py b/arkindex/process/migrations/0014_dataimport_selection.py
deleted file mode 100644
index 924758a4d41704ca4bbf78323bbc4785dee94c2e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0014_dataimport_selection.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# Generated by Django 2.2.13 on 2020-07-03 13:16
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0017_remove_elementtype_default_view'),
- ('process', '0013_create_model_workerrun'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='DataImportElement',
- fields=[
- ('id', models.UUIDField(
- default=uuid.uuid4,
- editable=False,
- primary_key=True,
- serialize=False,
- )),
- ('dataimport', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='dataimport_elements',
- to='process.DataImport',
- )),
- ('element', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='dataimport_elements',
- to='documents.Element',
- )),
- ],
- options={
- 'unique_together': {('dataimport', 'element')},
- },
- ),
- migrations.AddField(
- model_name='dataimport',
- name='elements',
- field=models.ManyToManyField(
- related_name='imports',
- through='process.DataImportElement',
- to='documents.Element',
- ),
- ),
- migrations.RemoveField(
- model_name='dataimport',
- name='use_selection',
- ),
- ]
diff --git a/arkindex/process/migrations/0015_clear_payload.py b/arkindex/process/migrations/0015_clear_payload.py
deleted file mode 100644
index 960b69bf385def557c40942ebf2c781046555f67..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0015_clear_payload.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Generated by Django 2.2.13 on 2020-07-07 13:49
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-# Converts DataImport payloads to fields, from all the keys found in existing payloads:
-# element_type → process.element_type
-# elt_type → process.element_type
-# folder_type → process.folder_type
-# element_id → process.element
-# folder_id → process.element
-# Many of those were not actually carried over by process.0011.
-# Ignored: corpus_id, ml_tools, pdf_engine, sha, repo_id
-
-PAYLOAD_TO_FIELDS = """
-UPDATE process_dataimport di
- SET element_type_id = type.id
- FROM documents_elementtype type
- WHERE
- di.payload IS NOT NULL
- AND di.element_type_id IS NULL
- AND type.corpus_id = di.corpus_id
- AND type.slug = di.payload->>'element_type';
-
-UPDATE process_dataimport di
- SET element_type_id = type.id
- FROM documents_elementtype type
- WHERE
- di.payload IS NOT NULL
- AND di.element_type_id IS NULL
- AND type.corpus_id = di.corpus_id
- AND type.slug = di.payload->>'elt_type';
-
-UPDATE process_dataimport di
- SET folder_type_id = type.id
- FROM documents_elementtype type
- WHERE
- di.payload IS NOT NULL
- AND type.corpus_id = di.corpus_id
- AND type.slug = di.payload->>'folder_type';
-
-UPDATE process_dataimport di
- SET element_id = element.id
- FROM documents_element element
- WHERE
- di.payload IS NOT NULL
- AND di.element_id IS NULL
- AND element.corpus_id = di.corpus_id
- AND element.id = (di.payload->>'element_id')::uuid;
-
-UPDATE process_dataimport di
- SET element_id = element.id
- FROM documents_element element
- WHERE
- di.payload IS NOT NULL
- AND di.element_id IS NULL
- AND element.corpus_id = di.corpus_id
- AND element.id = (di.payload->>'folder_id')::uuid;
-"""
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0017_remove_elementtype_default_view'),
- ('process', '0014_dataimport_selection'),
- ]
- atomic = False
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='folder_type',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='folder_imports',
- to='documents.ElementType',
- ),
- ),
- migrations.AlterField(
- model_name='dataimport',
- name='element_type',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='element_imports',
- to='documents.ElementType',
- ),
- ),
- migrations.RunSQL(PAYLOAD_TO_FIELDS),
- migrations.RemoveField(
- model_name='dataimport',
- name='payload',
- ),
- ]
diff --git a/arkindex/process/migrations/0016_new_jsonfield.py b/arkindex/process/migrations/0016_new_jsonfield.py
deleted file mode 100644
index 04e20974fb0ef8a4c54583411c33f83023657b0e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0016_new_jsonfield.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1 on 2020-08-10 14:47
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0015_clear_payload'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='workerversion',
- name='configuration',
- field=models.JSONField(),
- ),
- ]
diff --git a/arkindex/process/migrations/0017_dataimport_collection_id.py b/arkindex/process/migrations/0017_dataimport_collection_id.py
deleted file mode 100644
index 50b8514276734e2d5e72cc5d97cf00de20af35e3..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0017_dataimport_collection_id.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.2.13 on 2020-08-03 13:52
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0016_new_jsonfield'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='collection_id',
- field=models.PositiveIntegerField(blank=True, null=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0018_dataimport_build_entities.py b/arkindex/process/migrations/0018_dataimport_build_entities.py
deleted file mode 100644
index 3d19d58f2e1646621b5bc73a4a27581663a5ea8b..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0018_dataimport_build_entities.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1 on 2020-09-02 13:10
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0017_dataimport_collection_id'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='build_entities',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/process/migrations/0019_repository_type.py b/arkindex/process/migrations/0019_repository_type.py
deleted file mode 100644
index 571876d5614f45af68975ce77c7740a4a9c591ef..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0019_repository_type.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Generated by Django 3.1 on 2020-09-09 15:26
-
-from django.db import migrations, models
-
-WORKER_REPO_PREFIX = 'https://gitlab.com/teklia/workers/'
-
-
-def update_repository_types(apps, schema_editor):
- Repository = apps.get_model('process', 'Repository')
- # Defaults the repository to 'worker' type except if defined below
- Repository.objects \
- .filter(url__startswith=WORKER_REPO_PREFIX) \
- .update(type="iiif")
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0018_dataimport_build_entities'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='repository',
- name='type',
- field=models.CharField(
- default="iiif",
- max_length=10),
- preserve_default=False,
- ),
- migrations.RunPython(
- update_repository_types,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/process/migrations/0020_null_corpus_repo_dataimport.py b/arkindex/process/migrations/0020_null_corpus_repo_dataimport.py
deleted file mode 100644
index f5f70e31f879c3dac8ad8e3075465711f6f8d24f..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0020_null_corpus_repo_dataimport.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 3.1.1 on 2020-09-11 14:19
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0019_corpus_repository'),
- ('process', '0019_repository_type'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='dataimport',
- name='corpus',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='documents.corpus'),
- ),
- migrations.RemoveField(
- model_name='repository',
- name='corpus',
- ),
- ]
diff --git a/arkindex/process/migrations/0021_workerversion_docker_image_iid.py b/arkindex/process/migrations/0021_workerversion_docker_image_iid.py
deleted file mode 100644
index dd046817448b07c361bfec97bf7014dab2404872..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0021_workerversion_docker_image_iid.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 3.1.1 on 2020-09-22 09:09
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0020_null_corpus_repo_dataimport'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerversion',
- name='docker_image_iid',
- field=models.CharField(blank=True, max_length=80, null=True),
- ),
- migrations.AlterField(
- model_name='workerversion',
- name='docker_image',
- field=models.ForeignKey(blank=True, null=True, on_delete=models.deletion.SET_NULL, to='ponos.artifact'),
- ),
- ]
diff --git a/arkindex/process/migrations/0022_dataimport_load_children.py b/arkindex/process/migrations/0022_dataimport_load_children.py
deleted file mode 100644
index 56eaeaf613f61676e3cb05e536ce5066b09bff6a..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0022_dataimport_load_children.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.1 on 2020-09-21 07:31
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0021_workerversion_docker_image_iid'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='load_children',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/process/migrations/0023_workerversion_constraint.py b/arkindex/process/migrations/0023_workerversion_constraint.py
deleted file mode 100644
index b452fa1868de4f89dbd7940f244171a06d9ea1fe..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0023_workerversion_constraint.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 3.1.1 on 2020-09-24 10:57
-
-from django.db import migrations, models
-
-from arkindex.process.models import WorkerVersionState
-
-
-def update_invalid_workerversions(apps, schema_editor):
- WorkerVersion = apps.get_model('process', 'WorkerVersion')
- WorkerVersion.objects \
- .filter(state=WorkerVersionState.Available, docker_image__isnull=True) \
- .update(state=WorkerVersionState.Error)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0022_dataimport_load_children'),
- ]
-
- operations = [
- migrations.RunPython(
- update_invalid_workerversions,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AddConstraint(
- model_name='workerversion',
- constraint=models.CheckConstraint(
- check=~models.Q(state=WorkerVersionState.Available, docker_image_id=None),
- name='workerversion_available_requires_docker_image'
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0024_alter_worker_type.py b/arkindex/process/migrations/0024_alter_worker_type.py
deleted file mode 100644
index 10b55b816d7eb1becdc22338860aba6cead7868e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0024_alter_worker_type.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.4 on 2020-12-04 08:57
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0023_workerversion_constraint'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='worker',
- name='type',
- field=models.CharField(max_length=50),
- ),
- ]
diff --git a/arkindex/process/migrations/0025_dataimport_name.py b/arkindex/process/migrations/0025_dataimport_name.py
deleted file mode 100644
index 35f91a2c9aff72dd609dc5fc88aa9ec6bccbf3af..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0025_dataimport_name.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2020-12-04 09:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0024_alter_worker_type'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='name',
- field=models.CharField(blank=True, max_length=100, null=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0026_larger_content_type.py b/arkindex/process/migrations/0026_larger_content_type.py
deleted file mode 100644
index a3611197883e14fdc3f878215c797a1ed83e8811..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0026_larger_content_type.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-05 10:02
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0025_dataimport_name'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='datafile',
- name='content_type',
- field=models.CharField(max_length=120),
- ),
- ]
diff --git a/arkindex/process/migrations/0027_workers_rights.py b/arkindex/process/migrations/0027_workers_rights.py
deleted file mode 100644
index 25a52060bfc12c49da0b69333aaac959069608c1..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0027_workers_rights.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import django.db.models.deletion
-from django.contrib.contenttypes.models import ContentType
-from django.db import migrations, models
-
-
-def create_repository_admin(apps, schema_editor):
- Right = apps.get_model('users', 'Right')
- Repository = apps.get_model('process', 'Repository')
- # Define repositories admins from credentials used during the import
- Right.objects.bulk_create([
- Right(
- user_id=repo.credentials.user_id,
- content_type_id=ContentType.objects.get_for_model(Repository).id,
- content_id=repo.id,
- level=100
- ) for repo in Repository.objects.exclude(credentials__isnull=True)
- ])
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0026_larger_content_type'),
- ('users', '0011_corpus_rights'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='worker',
- name='public',
- field=models.BooleanField(default=False),
- ),
- migrations.AlterField(
- model_name='workerversion',
- name='worker',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='process.worker'),
- ),
- migrations.RunPython(
- create_repository_admin,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/process/migrations/0028_dataimport_max_lengths.py b/arkindex/process/migrations/0028_dataimport_max_lengths.py
deleted file mode 100644
index 6664e8c96513cbd384515849b84caa59948099cb..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0028_dataimport_max_lengths.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 3.1.6 on 2021-02-15 11:15
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0027_workers_rights'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='dataimport',
- name='best_class',
- field=models.CharField(blank=True, max_length=36, null=True),
- ),
- migrations.AlterField(
- model_name='dataimport',
- name='name_contains',
- field=models.CharField(blank=True, max_length=250, null=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0029_worker_activity.py b/arkindex/process/migrations/0029_worker_activity.py
deleted file mode 100644
index 30dd7888eae2ef6f00757526d3a9cb1753d4acab..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0029_worker_activity.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 3.1.5 on 2021-01-11 14:42
-
-import uuid
-
-import django.db.models.deletion
-import enumfields.fields
-from django.db import migrations, models
-
-import arkindex.process.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0026_metadata_worker_version'),
- ('process', '0028_dataimport_max_lengths'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='WorkerActivity',
- fields=[
- ('created', models.DateTimeField(auto_now_add=True)),
- ('updated', models.DateTimeField(auto_now=True)),
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('state', enumfields.fields.EnumField(default='queued', enum=arkindex.process.models.WorkerActivityState, max_length=10)),
- ('element', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activities', to='documents.element')),
- ('worker_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activities', to='process.workerversion')),
- ],
- options={
- 'unique_together': {('worker_version', 'element')},
- },
- ),
- ]
diff --git a/arkindex/process/migrations/0030_remove_mode_elements.py b/arkindex/process/migrations/0030_remove_mode_elements.py
deleted file mode 100644
index 9189c690e144daab97acd879924d74ce1490406e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0030_remove_mode_elements.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 3.1.6 on 2021-02-24 10:38
-
-from django.db import migrations
-from enumfields import Enum
-
-
-class OldDataImportMode(Enum):
- Elements = 'elements'
-
-
-def remove_mode_elements(apps, schema_editor):
- DataImport = apps.get_model('process', 'DataImport')
- DataImportElement = apps.get_model('process', 'DataImportElement')
-
- # Switch the enum to restore Elements, because the EnumField would otherwise not let us filter
- DataImport.mode.field.enum = OldDataImportMode
-
- DataImportElement.objects.filter(dataimport__mode=OldDataImportMode.Elements).delete()
- DataImport.objects.filter(mode=OldDataImportMode.Elements).delete()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0029_worker_activity'),
- ]
-
- operations = [
- migrations.RunPython(
- remove_mode_elements,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- )
- ]
diff --git a/arkindex/process/migrations/0031_dataimport_use_cache.py b/arkindex/process/migrations/0031_dataimport_use_cache.py
deleted file mode 100644
index 2dade8a420cbcb2590a7c13ed7a752f162a8042f..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0031_dataimport_use_cache.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.1.5 on 2021-03-30 08:49
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0030_remove_mode_elements'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='use_cache',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/process/migrations/0032_dataimport_activity_state.py b/arkindex/process/migrations/0032_dataimport_activity_state.py
deleted file mode 100644
index 3b7ddedea8c28efef3356a32f1a2deeae631e28c..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0032_dataimport_activity_state.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 3.1.5 on 2021-04-06 10:29
-
-import enumfields.fields
-from django.db import migrations
-
-import arkindex.process.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0031_dataimport_use_cache'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='activity_state',
- field=enumfields.fields.EnumField(default='disabled', enum=arkindex.process.models.ActivityState, max_length=32),
- ),
- ]
diff --git a/arkindex/process/migrations/0033_workeractivity_process.py b/arkindex/process/migrations/0033_workeractivity_process.py
deleted file mode 100644
index 1719e987d54088cdf62e114b36ce31490b6b664b..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0033_workeractivity_process.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Generated by Django 3.1.5 on 2021-05-07 07:48
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0032_dataimport_activity_state'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workeractivity',
- name='process',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='activities',
- to='process.dataimport'
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0034_worker_run_config.py b/arkindex/process/migrations/0034_worker_run_config.py
deleted file mode 100644
index 41cc0432e592545e1a2aa5aee7ca47e9b25a0a0f..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0034_worker_run_config.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Generated by Django 3.2.3 on 2021-07-09 13:01
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0033_workeractivity_process'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerrun',
- name='configuration',
- field=models.JSONField(default=dict),
- ),
- migrations.AddConstraint(
- model_name='workerrun',
- constraint=models.CheckConstraint(check=models.Q(('configuration__typeof', 'object')), name='worker_run_configuration_objects'),
- ),
- ]
diff --git a/arkindex/process/migrations/0035_corpus_version_cache.py b/arkindex/process/migrations/0035_corpus_version_cache.py
deleted file mode 100644
index 931de414dfdbd3708e520301eb361d7bdcc8d64e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0035_corpus_version_cache.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# Generated by Django 3.2.3 on 2021-08-31 07:53
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-def rebuild_reminder(apps, schema_editor):
- """
- Print a reminder to rebuild the cache manually if there is anything in the database.
- """
- Corpus = apps.get_model('documents', 'Corpus')
- if Corpus.objects.exists():
- print("Please run `arkindex cache_worker_versions` to fill the corpus worker versions cache.")
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0042_transcription_entity_confidence'),
- ('process', '0034_worker_run_config'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='CorpusWorkerVersion',
- fields=[
- ('id', models.UUIDField(
- default=uuid.uuid4,
- editable=False,
- primary_key=True,
- serialize=False
- )),
- ('corpus', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='worker_version_cache',
- to='documents.corpus',
- )),
- ('worker_version', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='corpus_cache',
- to='process.workerversion',
- )),
- ],
- options={
- 'unique_together': {('corpus', 'worker_version')},
- },
- ),
- migrations.AddField(
- model_name='workerversion',
- name='corpora',
- field=models.ManyToManyField(
- related_name='worker_versions',
- through='process.CorpusWorkerVersion',
- to='documents.Corpus',
- ),
- ),
- migrations.RunPython(
- code=rebuild_reminder,
- reverse_code=migrations.RunPython.noop,
- )
- ]
diff --git a/arkindex/process/migrations/0036_datafile_trashed.py b/arkindex/process/migrations/0036_datafile_trashed.py
deleted file mode 100644
index d7e4677ae995ba81ffe42c6fe70d3cd4470bf6c6..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0036_datafile_trashed.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.6 on 2021-09-09 07:21
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0035_corpus_version_cache'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='datafile',
- name='trashed',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/process/migrations/0037_workerversion_gpu_usage.py b/arkindex/process/migrations/0037_workerversion_gpu_usage.py
deleted file mode 100644
index 0d4850bb7af265a3d202abe55e43327c74e0aa5e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0037_workerversion_gpu_usage.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 3.2.6 on 2021-09-16 14:05
-
-import enumfields.fields
-from django.db import migrations
-
-import arkindex.process.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0036_datafile_trashed'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerversion',
- name='gpu_usage',
- field=enumfields.fields.EnumField(default='disabled', enum=arkindex.process.models.WorkerVersionGPUUsage, max_length=10, blank=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0038_dataimport_use_gpu.py b/arkindex/process/migrations/0038_dataimport_use_gpu.py
deleted file mode 100644
index 0cecc4d4443007143de6dd2b6e57f166e19add35..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0038_dataimport_use_gpu.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.6 on 2021-10-01 15:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0037_workerversion_gpu_usage'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='use_gpu',
- field=models.BooleanField(default=False, blank=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0039_worker_configuration.py b/arkindex/process/migrations/0039_worker_configuration.py
deleted file mode 100644
index c42e7074f78941365ffee0c9cc083f6568a35eb5..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0039_worker_configuration.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Generated by Django 3.2.5 on 2021-11-04 09:42
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-import arkindex.project.fields
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0038_dataimport_use_gpu'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='WorkerConfiguration',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('created', models.DateTimeField(auto_now_add=True)),
- ('updated', models.DateTimeField(auto_now=True)),
- ('name', models.CharField(max_length=250)),
- ('configuration', models.JSONField(blank=True, default=dict)),
- ('configuration_hash', arkindex.project.fields.MD5HashField(max_length=32)),
- ],
- ),
- migrations.RemoveConstraint(
- model_name='workerrun',
- name='worker_run_configuration_objects',
- ),
- migrations.RenameField(
- model_name='workerrun',
- old_name='configuration',
- new_name='old_configuration',
- ),
- migrations.AddConstraint(
- model_name='workerrun',
- constraint=models.CheckConstraint(check=models.Q(('old_configuration__typeof', 'object')), name='worker_run_old_configuration_objects'),
- ),
- migrations.AddField(
- model_name='workerconfiguration',
- name='worker',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='configurations', to='process.worker'),
- ),
- migrations.AddField(
- model_name='workeractivity',
- name='configuration',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='worker_activities', to='process.workerconfiguration'),
- ),
- migrations.AddField(
- model_name='workerrun',
- name='configuration',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='worker_runs', to='process.workerconfiguration'),
- ),
- migrations.AddConstraint(
- model_name='workerconfiguration',
- constraint=models.CheckConstraint(check=models.Q(('configuration__typeof', 'object')), name='worker_configuration_configuration_objects'),
- ),
- migrations.AlterUniqueTogether(
- name='workerconfiguration',
- unique_together={('worker', 'name'), ('worker', 'configuration_hash')},
- ),
- ]
diff --git a/arkindex/process/migrations/0040_use_worker_configuration.py b/arkindex/process/migrations/0040_use_worker_configuration.py
deleted file mode 100644
index 7c9c4769e82ef60a15a0adb873ed9f43593df3f3..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0040_use_worker_configuration.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# Generated by Django 3.2.5 on 2021-11-04 08:14
-
-from collections import defaultdict
-
-from django.db import migrations
-
-from arkindex.process.utils import hash_object
-
-
-def use_worker_configuration(apps, schema_editor):
- WorkerRun = apps.get_model('process', 'WorkerRun')
- WorkerConfiguration = apps.get_model('process', 'WorkerConfiguration')
-
- indexes = defaultdict(int)
-
- for worker_run in WorkerRun.objects.filter(configuration__isnull=True) \
- .exclude(old_configuration={}) \
- .select_related('version__worker'):
- worker = worker_run.version.worker
- worker_configuration, created = WorkerConfiguration.objects.get_or_create(
- worker=worker,
- configuration_hash=hash_object(worker_run.old_configuration),
- defaults={
- 'name': f'config for {worker.name} n°{indexes[worker.id]+1}',
- 'configuration': worker_run.old_configuration
- }
- )
- indexes[worker.id] += created
- worker_run.configuration = worker_configuration
- worker_run.save()
-
-
-def use_old_configuration(apps, schema_editor):
- WorkerRun = apps.get_model('process', 'WorkerRun')
-
- worker_runs = WorkerRun.objects.filter(configuration__isnull=False).select_related('configuration')
- for worker_run in worker_runs:
- worker_run.old_configuration = worker_run.configuration.configuration
- WorkerRun.objects.bulk_update(worker_runs, ['old_configuration'])
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0039_worker_configuration'),
- ]
-
- operations = [
- migrations.RunPython(
- use_worker_configuration,
- reverse_code=use_old_configuration,
- elidable=True,
- ),
- ]
diff --git a/arkindex/process/migrations/0041_remove_old_configuration.py b/arkindex/process/migrations/0041_remove_old_configuration.py
deleted file mode 100644
index b04a34706a855b858b6b70876fe2e77b7c306099..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0041_remove_old_configuration.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by Django 3.2.5 on 2021-11-04 08:14
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0040_use_worker_configuration'),
- ]
-
- operations = [
- migrations.RemoveConstraint(
- model_name='workerrun',
- name='worker_run_old_configuration_objects',
- ),
- migrations.RemoveField(
- model_name='workerrun',
- name='old_configuration',
- ),
- ]
diff --git a/arkindex/process/migrations/0042_alter_workeractivity_constraints.py b/arkindex/process/migrations/0042_alter_workeractivity_constraints.py
deleted file mode 100644
index 01a9f5c8e7a0fdf5ae316ebb08617cf0c822bbef..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0042_alter_workeractivity_constraints.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Generated by Django 3.2.5 on 2021-11-05 13:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0041_remove_old_configuration'),
- ]
-
- operations = [
- migrations.AlterUniqueTogether(
- name='workeractivity',
- unique_together=set(),
- ),
- migrations.AddConstraint(
- model_name='workeractivity',
- constraint=models.UniqueConstraint(condition=models.Q(('configuration__isnull', True)), fields=('worker_version', 'element'), name='worker_activity_unique_no_configuration'),
- ),
- migrations.AddConstraint(
- model_name='workeractivity',
- constraint=models.UniqueConstraint(condition=models.Q(('configuration__isnull', False)), fields=('worker_version', 'element', 'configuration'), name='worker_activity_unique_configuration'),
- ),
- ]
diff --git a/arkindex/process/migrations/0043_dataimport_template.py b/arkindex/process/migrations/0043_dataimport_template.py
deleted file mode 100644
index 0e9a3d1034470a1e00c0be73a1ae8253f9e2ddcb..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0043_dataimport_template.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.2.6 on 2021-11-18 12:13
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0042_alter_workeractivity_constraints'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='template',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='applications', to='process.dataimport'),
- ),
- ]
diff --git a/arkindex/process/migrations/0044_alter_workerconfiguration_configuration.py b/arkindex/process/migrations/0044_alter_workerconfiguration_configuration.py
deleted file mode 100644
index 1467a86be19ce330a446f7c7e6746143c8197e86..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0044_alter_workerconfiguration_configuration.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.6 on 2021-11-30 16:34
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0043_dataimport_template'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='workerconfiguration',
- name='configuration',
- field=models.JSONField(default=dict),
- ),
- ]
diff --git a/arkindex/process/migrations/0045_remove_dataimport_best_class.py b/arkindex/process/migrations/0045_remove_dataimport_best_class.py
deleted file mode 100644
index 07d089decb67005541753d811d67c153a889bdcb..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0045_remove_dataimport_best_class.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 3.2.6 on 2021-12-14 14:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0044_alter_workerconfiguration_configuration'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='dataimport',
- name='best_class',
- ),
- ]
diff --git a/arkindex/process/migrations/0046_workertype_alter_worker_type.py b/arkindex/process/migrations/0046_workertype_alter_worker_type.py
deleted file mode 100644
index 19062cf5e9e9b3b2fb597b65943e2444f5416de4..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0046_workertype_alter_worker_type.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Generated by Django 4.0.2 on 2022-04-07 11:43
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-def update_worker_types(apps, schema_editor):
- Worker = apps.get_model('process', 'Worker')
- WorkerType = apps.get_model('process', 'WorkerType')
-
- # Get list of current worker types
- current_types = Worker.objects.values('type').distinct()
- created_types = WorkerType.objects.bulk_create(
- [WorkerType(slug=type_slug['type'], display_name=type_slug['type'].capitalize()) for type_slug in current_types]
- )
- for worker_type in created_types:
- Worker.objects.filter(type=worker_type.slug).update(type_fk=worker_type.id)
-
-
-def retrieve_worker_type_slugs(apps, schema_editor):
- Worker = apps.get_model('process', 'Worker')
- for worker in Worker.objects.all():
- worker.type = worker.type_fk.slug
- worker.save()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0045_remove_dataimport_best_class'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='WorkerType',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('slug', models.CharField(max_length=100, unique=True)),
- ('display_name', models.CharField(max_length=100)),
- ('created', models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now)),
- ('updated', models.DateTimeField(auto_now=True))
- ],
- ),
- migrations.AddField(
- model_name='worker',
- name='type_fk',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='type', to='process.workertype'),
- ),
- migrations.AlterField(
- model_name="worker",
- name="type",
- field=models.CharField(max_length=50, null=True),
- ),
- migrations.RunPython(
- update_worker_types,
- reverse_code=retrieve_worker_type_slugs
- ),
- migrations.RemoveField(
- model_name='worker',
- name='type',
- ),
- migrations.RenameField(
- model_name='worker',
- old_name='type_fk',
- new_name='type',
- ),
- migrations.AlterField(
- model_name="worker",
- name="type",
- field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='type', to='process.workertype'),
- ),
- migrations.AlterField(
- model_name="workertype",
- name="created",
- field=models.DateTimeField(auto_now_add=True)
- )
- ]
diff --git a/arkindex/process/migrations/0047_workerversion_model_usage.py b/arkindex/process/migrations/0047_workerversion_model_usage.py
deleted file mode 100644
index df9392b63bddcf1078c4e4e3d16a8f045f7e9d3e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0047_workerversion_model_usage.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.0.2 on 2022-05-03 15:12
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0046_workertype_alter_worker_type'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerversion',
- name='model_usage',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/process/migrations/0048_workerrun_model_version.py b/arkindex/process/migrations/0048_workerrun_model_version.py
deleted file mode 100644
index 8fc0e95c201b39886a08ba80ec9c45da124578f1..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0048_workerrun_model_version.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 4.0.2 on 2022-05-03 16:14
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0004_modelversion_archive_hash'),
- ('process', '0047_workerversion_model_usage'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerrun',
- name='model_version',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='worker_runs', to='training.modelversion'),
- ),
- ]
diff --git a/arkindex/process/migrations/0049_dataimport_s3.py b/arkindex/process/migrations/0049_dataimport_s3.py
deleted file mode 100644
index 27594f2899e55a9d98f4fe3bf20fc11421114ffa..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0049_dataimport_s3.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 4.0.4 on 2022-06-14 13:52
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0048_workerrun_model_version'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='bucket_name',
- field=models.CharField(blank=True, max_length=63, null=True, validators=[django.core.validators.MinLengthValidator(3)]),
- ),
- migrations.AddField(
- model_name='dataimport',
- name='prefix',
- field=models.CharField(blank=True, max_length=1024, null=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0050_workerrun_summary.py b/arkindex/process/migrations/0050_workerrun_summary.py
deleted file mode 100644
index f2aa8b5a56e21d28b78bbb56f03fe63d6f79a7d4..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0050_workerrun_summary.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Generated by Django 4.0.2 on 2022-06-13 12:40
-
-from django.db import migrations, models
-from django.db.models.query import Prefetch
-
-
-def generate_worker_run_summaries(apps, schema_editor):
- WorkerRun = apps.get_model('process', 'WorkerRun')
- GitRef = apps.get_model('process', 'GitRef')
- runs = WorkerRun.objects.select_related(
- 'version__worker',
- 'version__revision',
- 'model_version__model',
- 'configuration'
- ).only(
- # Required to make joins and for the bulk update
- 'id',
- 'version_id',
- 'version__worker__name',
- # Required for the prefetch_related. Using `revision_id` instead of `revision__id` causes Django to select the entire revision anyway
- 'version__revision__id',
- 'model_version_id',
- 'model_version__model__name',
- 'configuration__name'
- ).prefetch_related(
- Prefetch(
- 'version__revision__refs',
- # revision_id is required to make the link between refs and revisions
- queryset=GitRef.objects.only('revision_id', 'name')
- )
- )
-
- for run in runs:
- summary_text = f"Worker {run.version.worker.name} @ "
- git_ref_names = run.version.revision.refs.values_list('name', flat=True)
- if len(git_ref_names) > 0:
- summary_text += ", ".join(git_ref_names)
- else:
- summary_text += str(run.version.id)[0:6]
-
- if run.model_version:
- summary_text += f" with model {run.model_version.model.name} @ {str(run.model_version.id)[0:6]}"
-
- if run.configuration:
- summary_text += f" using configuration '{run.configuration.name}'"
-
- run.summary = summary_text
- WorkerRun.objects.bulk_update(runs, ['summary'])
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0049_dataimport_s3'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerrun',
- name='summary',
- field=models.TextField(null=True),
- ),
- migrations.RunPython(
- generate_worker_run_summaries,
- reverse_code=migrations.RunPython.noop
- ),
- migrations.AlterField(
- model_name='workerrun',
- name='summary',
- field=models.TextField(),
- ),
- ]
diff --git a/arkindex/process/migrations/0051_workerconfiguration_archived.py b/arkindex/process/migrations/0051_workerconfiguration_archived.py
deleted file mode 100644
index 2f5659cd6b8fc058f8d58af3c8d5c5e745563769..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0051_workerconfiguration_archived.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.0.5 on 2022-07-12 13:10
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0050_workerrun_summary'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workerconfiguration',
- name='archived',
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/process/migrations/0052_dataimport_test_and_train_folders_fks.py b/arkindex/process/migrations/0052_dataimport_test_and_train_folders_fks.py
deleted file mode 100644
index 4ecbfba5e7ac6c5fbc696ecfa408f77869018998..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0052_dataimport_test_and_train_folders_fks.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Generated by Django 4.1a1 on 2022-07-07 14:14
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("documents", "0057_entity_list_index"),
- ("process", "0051_workerconfiguration_archived"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="dataimport",
- name="test_folder",
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="test_folder_processes",
- to="documents.element",
- ),
- ),
- migrations.AddField(
- model_name="dataimport",
- name="train_folder",
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="train_folder_processes",
- to="documents.element",
- ),
- ),
- migrations.AddField(
- model_name='dataimport',
- name='validation_folder',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name='validation_folder_processes',
- to='documents.element'
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0053_remove_dataimport_build_entities.py b/arkindex/process/migrations/0053_remove_dataimport_build_entities.py
deleted file mode 100644
index ae4a1588fbe9a235717aea452ba9037a8824c4a5..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0053_remove_dataimport_build_entities.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.2 on 2022-07-25 15:22
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0052_dataimport_test_and_train_folders_fks'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='dataimport',
- name='build_entities',
- ),
- ]
diff --git a/arkindex/process/migrations/0054_alter_workertype_slug.py b/arkindex/process/migrations/0054_alter_workertype_slug.py
deleted file mode 100644
index f409fe986234b8be72d2fa191be129dba19e6e20..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0054_alter_workertype_slug.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-01 10:12
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0053_remove_dataimport_build_entities'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='workertype',
- name='slug',
- field=models.SlugField(max_length=100, unique=True),
- ),
- ]
diff --git a/arkindex/process/migrations/0055_remove_repository_type_iiif.py b/arkindex/process/migrations/0055_remove_repository_type_iiif.py
deleted file mode 100644
index f4958ffbe08933c7e3b5be0a9b25da3630a3af99..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0055_remove_repository_type_iiif.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Generated by Django 4.0.2 on 2022-07-26 14:19
-
-from django.db import migrations
-from enumfields import Enum
-
-
-class OldRepositoryType(Enum):
- IIIF = 'iiif'
-
-
-def remove_git_iiif(apps, schema_editor):
- Repository = apps.get_model('process', 'Repository')
- Repository.type.field.enum = OldRepositoryType
- Repository.objects.filter(type=OldRepositoryType.IIIF).delete()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0054_alter_workertype_slug'),
- ]
-
- operations = [
- migrations.RunPython(
- remove_git_iiif,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/process/migrations/0056_remove_repository_type.py b/arkindex/process/migrations/0056_remove_repository_type.py
deleted file mode 100644
index 904a3323b1215132772c84d11dd487b058314e7f..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0056_remove_repository_type.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-03 16:36
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0055_remove_repository_type_iiif'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='repository',
- name='type'
- )
- ]
diff --git a/arkindex/process/migrations/0057_dataimport_model.py b/arkindex/process/migrations/0057_dataimport_model.py
deleted file mode 100644
index f836997a5f47fc51b421215801fafc428d8889ce..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0057_dataimport_model.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-03 15:11
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0004_modelversion_archive_hash'),
- ('process', '0056_remove_repository_type'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='dataimport',
- name='model',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='training_processes', to='training.model'),
- ),
- ]
diff --git a/arkindex/process/migrations/0058_merge_file_imports.py b/arkindex/process/migrations/0058_merge_file_imports.py
deleted file mode 100644
index 53f4ba85d547efbc7cec6d5f14c43a8a90df981e..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0058_merge_file_imports.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-23 15:46
-
-from django.db import migrations
-from enumfields import Enum
-
-
-class OldImportModes(Enum):
- Images = 'images'
- PDF = 'pdf'
- Files = 'files'
-
-
-def update_process_mode(apps, schema_editor):
- DataImport = apps.get_model('process', 'DataImport')
- DataImport.mode.field.enum = OldImportModes
- updated_processes = DataImport.objects.filter(mode__in=OldImportModes)
- updated_processes.update(mode=OldImportModes.Files)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0057_dataimport_model'),
- ]
-
- operations = [
- migrations.RunPython(
- update_process_mode,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/process/migrations/0059_rename_dataimport_to_process.py b/arkindex/process/migrations/0059_rename_dataimport_to_process.py
deleted file mode 100644
index ba9b5f036247ac79890c2ecb1defe2af9f1a53aa..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0059_rename_dataimport_to_process.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# Generated by Django 4.0.4 on 2022-08-31 09:14
-
-import django.core.validators
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0004_modelversion_archive_hash'),
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('ponos', '0032_stringify_json'),
- ('documents', '0058_remove_corpus_repository'),
- ('process', '0058_merge_file_imports'),
- ]
-
- operations = [
- migrations.RenameModel(
- old_name='DataImport',
- new_name='Process',
- ),
- migrations.RenameModel(
- old_name='DataImportElement',
- new_name='ProcessElement',
- ),
- migrations.RenameField(
- model_name='processelement',
- old_name='dataimport',
- new_name='process',
- ),
- migrations.RenameField(
- model_name='workerrun',
- old_name='dataimport',
- new_name='process',
- ),
- migrations.AlterField(
- model_name='process',
- name='corpus',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='documents.corpus'),
- ),
- migrations.AlterField(
- model_name='process',
- name='creator',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='processes', to=settings.AUTH_USER_MODEL),
- ),
- migrations.AlterField(
- model_name='process',
- name='elements',
- field=models.ManyToManyField(related_name='processes', through='process.ProcessElement', to='documents.element'),
- ),
- migrations.AlterField(
- model_name='process',
- name='files',
- field=models.ManyToManyField(related_name='processes', to='process.datafile'),
- ),
- migrations.AlterField(
- model_name='process',
- name='revision',
- field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='process.revision'),
- ),
- migrations.AlterField(
- model_name='process',
- name='versions',
- field=models.ManyToManyField(related_name='processes', through='process.WorkerRun', to='process.workerversion'),
- ),
- migrations.AlterField(
- model_name='processelement',
- name='element',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='process_elements', to='documents.element'),
- ),
- migrations.AlterField(
- model_name='processelement',
- name='process',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='process_elements', to='process.process'),
- ),
- migrations.AlterModelOptions(
- name='process',
- options={'ordering': ['corpus', '-created'], 'verbose_name_plural': 'processes'},
- ),
- ]
diff --git a/arkindex/process/migrations/0060_remove_repository_provider_name.py b/arkindex/process/migrations/0060_remove_repository_provider_name.py
deleted file mode 100644
index 9fa7c26a5353737e59278df85b1d4ef851cdcbcd..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0060_remove_repository_provider_name.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.7 on 2022-10-18 15:51
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0059_rename_dataimport_to_process'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='repository',
- name='provider_name',
- ),
- ]
diff --git a/arkindex/process/migrations/0061_workeractivity_updated_triggers.py b/arkindex/process/migrations/0061_workeractivity_updated_triggers.py
deleted file mode 100644
index 65b6dd3877cae2f1c04de85ba801f8b1ce40af07..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0061_workeractivity_updated_triggers.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Generated by Django 4.0.7 on 2022-10-19 16:05
-
-from django.db import migrations, models
-
-import pgtrigger.compiler
-import pgtrigger.migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0060_remove_repository_provider_name'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='workeractivity',
- name='updated',
- field=models.DateTimeField(auto_now_add=True),
- ),
- pgtrigger.migrations.AddTrigger(
- model_name='workeractivity',
- trigger=pgtrigger.compiler.Trigger(
- name='update_workeractivity_updated',
- sql=pgtrigger.compiler.UpsertTriggerSql(
- func='NEW.updated = now(); RETURN NEW;',
- hash='1e5a8fa0718f420e6cd4f2a31434cd39a9c9bc67',
- operation='UPDATE',
- pgid='pgtrigger_update_workeractivity_updated_f2812',
- table='process_workeractivity',
- when='BEFORE',
- )
- ),
- ),
- pgtrigger.migrations.AddTrigger(
- model_name='workeractivity',
- trigger=pgtrigger.compiler.Trigger(
- name='read_only_workeractivity_updated',
- sql=pgtrigger.compiler.UpsertTriggerSql(
- condition='WHEN (OLD."updated" IS DISTINCT FROM (NEW."updated"))',
- func="RAISE EXCEPTION 'pgtrigger: Cannot update rows from % table', TG_TABLE_NAME;",
- hash='6276c6971a1d2669659e407418e2db1fa7dc6965',
- operation='UPDATE',
- pgid='pgtrigger_read_only_workeractivity_updated_a80ab',
- table='process_workeractivity',
- when='BEFORE',
- )
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0062_workeractivity_started.py b/arkindex/process/migrations/0062_workeractivity_started.py
deleted file mode 100644
index a07c75a07e509f6ff5f6c5477263a0e05cf7149c..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0062_workeractivity_started.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Generated by Django 4.0.4 on 2022-12-05 09:44
-
-from django.db import migrations, models
-
-from arkindex.process.models import WorkerActivityState
-
-
-def set_started_on_started(apps, schema_editor):
- """
- In case this migration runs while some processes were running, some activities might be in a `started` state
- and we therefore need to set their start time before we can apply the new check constraint.
- We do know their start time though: their last update time will be the time where they were set to `started`.
- """
- WorkerActivity = apps.get_model('process', 'WorkerActivity')
- WorkerActivity.objects.filter(state=WorkerActivityState.Started).update(started=models.F('updated'))
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0061_workeractivity_updated_triggers'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='workeractivity',
- name='started',
- field=models.DateTimeField(blank=True, null=True),
- ),
- migrations.RunPython(
- set_started_on_started,
- reverse_code=migrations.RunPython.noop,
- ),
- migrations.AddConstraint(
- model_name='workeractivity',
- constraint=models.CheckConstraint(
- check=~models.Q(state=WorkerActivityState.Started) | models.Q(started__isnull=False),
- name='worker_activity_started_requires_started',
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0063_workeractivity_process_not_null.py b/arkindex/process/migrations/0063_workeractivity_process_not_null.py
deleted file mode 100644
index 8174e3ade5c2e4148f9c79ef05593a7fdcab9c51..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0063_workeractivity_process_not_null.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# Generated by Django 4.1.4 on 2023-01-09 15:42
-
-from django.db import migrations, models
-
-
-def check_no_null_process_id(apps, schema_editor):
- WorkerActivity = apps.get_model('process', 'WorkerActivity')
- assert not WorkerActivity.objects.filter(process_id=None).exists(), (
- 'Some WorkerActivities exist without a `process_id`. '
- 'Please either link them to a process or delete them before running this migration.'
- )
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0062_workeractivity_started'),
- ]
-
- operations = [
- migrations.RunPython(
- check_no_null_process_id,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AlterField(
- model_name='workeractivity',
- name='process',
- field=models.ForeignKey(
- on_delete=models.CASCADE,
- related_name='activities',
- to='process.Process',
- ),
- ),
- ]
diff --git a/arkindex/process/migrations/0064_local_process_constraints.py b/arkindex/process/migrations/0064_local_process_constraints.py
deleted file mode 100644
index e17ef131cc5526e369ecf820abaa8f889ddd9ff3..0000000000000000000000000000000000000000
--- a/arkindex/process/migrations/0064_local_process_constraints.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Generated by Django 4.1.5 on 2023-02-14 10:16
-
-from django.db import migrations, models
-
-from arkindex.process.models import ProcessMode
-
-
-def remove_corpus_on_repository_processes(apps, schema_editor):
- """
- Processes that were ran on IIIF repositories had a corpus linked to them.
- This unsets the corpus on those as we now only support worker imports
- and repository processes should no longer have a corpus.
- """
- Process = apps.get_model('process', 'Process')
- Process.objects.filter(mode=ProcessMode.Repository).exclude(corpus=None).update(corpus=None)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('process', '0063_workeractivity_process_not_null'),
- ]
-
- operations = [
- migrations.RunPython(
- remove_corpus_on_repository_processes,
- reverse_code=migrations.RunPython.noop,
- elidable=True,
- ),
- migrations.AddConstraint(
- model_name='process',
- constraint=models.CheckConstraint(
- check=models.Q(mode__in=(ProcessMode.Local, ProcessMode.Repository)) ^ ~models.Q(corpus=None),
- name='check_process_corpus',
- violation_error_message='Local and repository processes cannot have a corpus, and other modes must have one set.',
- ),
- ),
- migrations.AddConstraint(
- model_name='process',
- constraint=models.CheckConstraint(
- check=~models.Q(mode=ProcessMode.Local) | models.Q(workflow=None),
- name='local_process_no_workflow',
- violation_error_message='Local processes cannot be started.',
- ),
- ),
- migrations.AddConstraint(
- model_name='process',
- constraint=models.UniqueConstraint(
- models.F('creator'),
- condition=models.Q(mode=ProcessMode.Local),
- name='unique_local_process',
- violation_error_message='Only one local process is allowed per user.',
- ),
- ),
- ]
diff --git a/arkindex/training/migrations/0001_initial.py b/arkindex/training/migrations/0001_initial.py
index df26b05b33600a6cdd3e40ba313901cf54561a7b..56ad1bc37d92ed6b2855979b3b2aa04f6a1c291a 100644
--- a/arkindex/training/migrations/0001_initial.py
+++ b/arkindex/training/migrations/0001_initial.py
@@ -1,11 +1,14 @@
-# Generated by Django 4.0.2 on 2022-03-16 09:50
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import uuid
+import django.contrib.postgres.fields
+import django.core.validators
import django.db.models.deletion
import enumfields.fields
from django.db import migrations, models
+import arkindex.project.aws
import arkindex.project.fields
import arkindex.training.models
@@ -15,10 +18,32 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
- ('process', '0045_remove_dataimport_best_class'),
+ ('process', '0001_initial'),
+ ('documents', '0002_initial'),
]
operations = [
+ migrations.CreateModel(
+ name='Dataset',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('name', models.CharField(max_length=100, validators=[django.core.validators.MinLengthValidator(1)])),
+ ('description', models.TextField(validators=[django.core.validators.MinLengthValidator(1)])),
+ ('state', enumfields.fields.EnumField(default='open', enum=arkindex.training.models.DatasetState, max_length=10)),
+ ('sets', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=50, validators=[django.core.validators.MinLengthValidator(1)]), default=arkindex.training.models.default_sets, size=None, validators=[django.core.validators.MinLengthValidator(1), arkindex.training.models.validate_unique_set_names])),
+ ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='datasets', to='documents.corpus')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='MetricKey',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=100)),
+ ('mode', enumfields.fields.EnumField(default='series', enum=arkindex.training.models.MetricMode, max_length=10)),
+ ],
+ ),
migrations.CreateModel(
name='Model',
fields=[
@@ -28,7 +53,7 @@ class Migration(migrations.Migration):
('name', models.CharField(max_length=100, unique=True)),
('description', models.TextField(default='')),
('public', models.BooleanField(default=False)),
- ('compatible_workers', models.ManyToManyField(related_name='models', to='process.Worker')),
+ ('compatible_workers', models.ManyToManyField(related_name='models', to='process.worker')),
],
options={
'abstract': False,
@@ -41,16 +66,39 @@ class Migration(migrations.Migration):
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('description', models.TextField(default='')),
- ('tag', models.CharField(blank=True, max_length=50, null=True)),
+ ('tag', models.CharField(default=None, max_length=50, null=True)),
('state', enumfields.fields.EnumField(default='created', enum=arkindex.training.models.ModelVersionState, max_length=10)),
- ('hash', arkindex.project.fields.MD5HashField(max_length=32)),
- ('size', models.PositiveIntegerField(help_text='file size in bytes')),
+ ('hash', arkindex.project.fields.MD5HashField(blank=True, help_text="hash of the content of the archive which contains the model version's data", max_length=32, null=True)),
+ ('archive_hash', arkindex.project.fields.MD5HashField(blank=True, help_text="hash of the archive which contains the model version's data", max_length=32, null=True)),
+ ('size', models.PositiveIntegerField(blank=True, help_text='file size in bytes', null=True)),
('configuration', models.JSONField(default=dict)),
('model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='training.model')),
('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='training.modelversion')),
],
- options={
- 'unique_together': {('model', 'tag')},
- },
+ bases=(arkindex.project.aws.S3FileMixin, models.Model),
+ ),
+ migrations.CreateModel(
+ name='MetricValue',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('value', models.FloatField()),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('step', models.PositiveIntegerField(blank=True, null=True)),
+ ('metric', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='values', to='training.metrickey')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='metrickey',
+ name='model_version',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metrics', to='training.modelversion'),
+ ),
+ migrations.CreateModel(
+ name='DatasetElement',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('set', models.CharField(max_length=50, validators=[django.core.validators.MinLengthValidator(1)])),
+ ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='dataset_elements', to='training.dataset')),
+ ('element', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='dataset_elements', to='documents.element')),
+ ],
),
]
diff --git a/arkindex/training/migrations/0002_alter_modelversion_tag.py b/arkindex/training/migrations/0002_alter_modelversion_tag.py
deleted file mode 100644
index 156b5506b4db00cdef06fc408b0513bf6da4807d..0000000000000000000000000000000000000000
--- a/arkindex/training/migrations/0002_alter_modelversion_tag.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.0.2 on 2022-03-23 14:06
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0001_initial'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='modelversion',
- name='tag',
- field=models.CharField(blank=True, default=None, max_length=50, null=True),
- ),
- ]
diff --git a/arkindex/training/migrations/0002_initial.py b/arkindex/training/migrations/0002_initial.py
new file mode 100644
index 0000000000000000000000000000000000000000..5428664be49d9827699c2c350f961371eee0f157
--- /dev/null
+++ b/arkindex/training/migrations/0002_initial.py
@@ -0,0 +1,63 @@
+# Generated by Django 4.1.7 on 2023-05-29 11:49
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('documents', '0003_initial'),
+ ('training', '0001_initial'),
+ ('ponos', '0001_initial'),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='dataset',
+ name='creator',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='datasets', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='dataset',
+ name='elements',
+ field=models.ManyToManyField(related_name='datasets', through='training.DatasetElement', to='documents.element'),
+ ),
+ migrations.AddField(
+ model_name='dataset',
+ name='task',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset', to='ponos.task'),
+ ),
+ migrations.AddConstraint(
+ model_name='modelversion',
+ constraint=models.UniqueConstraint(fields=('model', 'tag'), name='modelversion_unique_tag'),
+ ),
+ migrations.AddConstraint(
+ model_name='modelversion',
+ constraint=models.UniqueConstraint(condition=models.Q(('hash__isnull', False)), fields=('model', 'hash'), name='modelversion_unique_hash'),
+ ),
+ migrations.AddConstraint(
+ model_name='metricvalue',
+ constraint=models.UniqueConstraint(condition=models.Q(('step__isnull', False)), fields=('metric', 'step'), name='metric_unique_step'),
+ ),
+ migrations.AddConstraint(
+ model_name='metricvalue',
+ constraint=models.UniqueConstraint(condition=models.Q(('step__isnull', True)), fields=('metric', 'created'), name='metric_unique_no_step'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='metrickey',
+ unique_together={('name', 'model_version')},
+ ),
+ migrations.AddConstraint(
+ model_name='datasetelement',
+ constraint=models.UniqueConstraint(fields=('dataset', 'element', 'set'), name='unique_dataset_elements'),
+ ),
+ migrations.AddConstraint(
+ model_name='dataset',
+ constraint=models.UniqueConstraint(fields=('corpus', 'name'), name='unique_dataset_names'),
+ ),
+ ]
diff --git a/arkindex/training/migrations/0003_alter_modelversion_unique_together_and_more.py b/arkindex/training/migrations/0003_alter_modelversion_unique_together_and_more.py
deleted file mode 100644
index 5a8c7217265d8bbe1772736d0992667b9ba17942..0000000000000000000000000000000000000000
--- a/arkindex/training/migrations/0003_alter_modelversion_unique_together_and_more.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 4.0.2 on 2022-03-31 08:51
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0002_alter_modelversion_tag'),
- ]
-
- operations = [
- migrations.AlterUniqueTogether(
- name='modelversion',
- unique_together={('model', 'tag')},
- ),
- migrations.AlterField(
- model_name='modelversion',
- name='tag',
- field=models.CharField(default=None, max_length=50, null=True),
- ),
- migrations.AlterUniqueTogether(
- name='modelversion',
- unique_together={('model', 'tag'), ('model', 'hash')},
- ),
- ]
diff --git a/arkindex/training/migrations/0004_modelversion_archive_hash.py b/arkindex/training/migrations/0004_modelversion_archive_hash.py
deleted file mode 100644
index 32de137a13ca19d6374918072c62c1abb26fffb5..0000000000000000000000000000000000000000
--- a/arkindex/training/migrations/0004_modelversion_archive_hash.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 4.0.2 on 2022-04-14 11:02
-
-from django.db import migrations
-
-import arkindex.project.fields
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0003_alter_modelversion_unique_together_and_more'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='modelversion',
- name='archive_hash',
- field=arkindex.project.fields.MD5HashField(default='00000000000000000000000000000000', help_text="hash of the archive which contains the model version's data", max_length=32),
- preserve_default=False,
- ),
- migrations.AlterField(
- model_name='modelversion',
- name='hash',
- field=arkindex.project.fields.MD5HashField(help_text="hash of the content of the archive which contains the model version's data", max_length=32),
- ),
- ]
diff --git a/arkindex/training/migrations/0005_metrics_metrickey_metricvalue.py b/arkindex/training/migrations/0005_metrics_metrickey_metricvalue.py
deleted file mode 100644
index d3df988424f702263b429244176735a19250a5c0..0000000000000000000000000000000000000000
--- a/arkindex/training/migrations/0005_metrics_metrickey_metricvalue.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Generated by Django 4.0.4 on 2022-12-09 14:59
-
-import uuid
-
-import django.db.models.deletion
-import enumfields.fields
-from django.db import migrations, models
-
-import arkindex.training.models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0004_modelversion_archive_hash'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='MetricKey',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=100)),
- ('mode', enumfields.fields.EnumField(default='series', enum=arkindex.training.models.MetricMode, max_length=10)),
- ('model_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metrics', to='training.modelversion')),
- ],
- ),
- migrations.CreateModel(
- name='MetricValue',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('value', models.FloatField()),
- ('created', models.DateTimeField(auto_now_add=True)),
- ('step', models.PositiveIntegerField(blank=True, null=True)),
- ('metric', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='values', to='training.metrickey')),
- ],
- ),
- migrations.AddConstraint(
- model_name='metricvalue',
- constraint=models.UniqueConstraint(condition=models.Q(('step__isnull', False)), fields=('metric', 'step'), name='metric_unique_step'),
- ),
- migrations.AddConstraint(
- model_name='metricvalue',
- constraint=models.UniqueConstraint(condition=models.Q(('step__isnull', True)), fields=('metric', 'created'), name='metric_unique_no_step'),
- ),
- migrations.AlterUniqueTogether(
- name='metrickey',
- unique_together={('name', 'model_version')},
- ),
- ]
diff --git a/arkindex/training/migrations/0006_alter_modelversion_unique_together_and_more.py b/arkindex/training/migrations/0006_alter_modelversion_unique_together_and_more.py
deleted file mode 100644
index bc0f3e97dc85293de869fd0d6daf1543b49239b3..0000000000000000000000000000000000000000
--- a/arkindex/training/migrations/0006_alter_modelversion_unique_together_and_more.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Generated by Django 4.1.4 on 2023-01-03 13:34
-
-from django.db import migrations, models
-
-import arkindex.project.fields
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('training', '0005_metrics_metrickey_metricvalue'),
- ]
-
- operations = [
- migrations.AlterUniqueTogether(
- name='modelversion',
- unique_together=set(),
- ),
- migrations.AlterField(
- model_name='modelversion',
- name='archive_hash',
- field=arkindex.project.fields.MD5HashField(blank=True, help_text="hash of the archive which contains the model version's data", max_length=32, null=True),
- ),
- migrations.AlterField(
- model_name='modelversion',
- name='hash',
- field=arkindex.project.fields.MD5HashField(blank=True, help_text="hash of the content of the archive which contains the model version's data", max_length=32, null=True),
- ),
- migrations.AlterField(
- model_name='modelversion',
- name='size',
- field=models.PositiveIntegerField(blank=True, help_text='file size in bytes', null=True),
- ),
- migrations.AddConstraint(
- model_name='modelversion',
- constraint=models.UniqueConstraint(fields=('model', 'tag'), name='modelversion_unique_tag'),
- ),
- migrations.AddConstraint(
- model_name='modelversion',
- constraint=models.UniqueConstraint(condition=models.Q(('hash__isnull', False)), fields=('model', 'hash'), name='modelversion_unique_hash'),
- ),
- ]
diff --git a/arkindex/training/migrations/0007_datasets.py b/arkindex/training/migrations/0007_datasets.py
deleted file mode 100644
index ba8b3da58c91790fc7fae3cc71e164de588cdeac..0000000000000000000000000000000000000000
--- a/arkindex/training/migrations/0007_datasets.py
+++ /dev/null
@@ -1,81 +0,0 @@
-# Generated by Django 4.1.7 on 2023-05-15 16:04
-
-import uuid
-
-from django.conf import settings
-from django.contrib.postgres.fields import ArrayField
-from django.core.validators import MinLengthValidator
-from django.db import migrations, models
-from enumfields import EnumField
-
-from arkindex.training.models import DatasetState, default_sets, validate_unique_set_names
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0064_alter_entity_type_alter_entityrole_child_type_and_more'),
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('ponos', '0039_keep_task_gpu'),
- ('training', '0006_alter_modelversion_unique_together_and_more'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Dataset',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('created', models.DateTimeField(auto_now_add=True)),
- ('updated', models.DateTimeField(auto_now=True)),
- ('name', models.CharField(max_length=100, validators=[MinLengthValidator(1)])),
- ('description', models.TextField(validators=[MinLengthValidator(1)])),
- ('state', EnumField(default='open', enum=DatasetState, max_length=10)),
- ('sets', ArrayField(base_field=models.CharField(max_length=50, validators=[MinLengthValidator(1)]), size=None, validators=[MinLengthValidator(1), validate_unique_set_names], default=default_sets)),
- ('corpus', models.ForeignKey(on_delete=models.DO_NOTHING, related_name='datasets', to='documents.corpus')),
- ('creator', models.ForeignKey(on_delete=models.DO_NOTHING, related_name='datasets', to=settings.AUTH_USER_MODEL)),
- ],
- ),
- migrations.CreateModel(
- name='DatasetElement',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('set', models.CharField(max_length=50, validators=[MinLengthValidator(1)])),
- ('dataset', models.ForeignKey(on_delete=models.DO_NOTHING, related_name='dataset_elements', to='training.dataset')),
- ('element', models.ForeignKey(on_delete=models.DO_NOTHING, related_name='dataset_elements', to='documents.element')),
- ],
- ),
- migrations.AddField(
- model_name='dataset',
- name='elements',
- field=models.ManyToManyField(
- related_name='datasets',
- through='training.DatasetElement',
- to='documents.element',
- ),
- ),
- migrations.AddField(
- model_name='dataset',
- name='task',
- field=models.ForeignKey(
- blank=True,
- null=True,
- on_delete=models.SET_NULL,
- related_name='dataset',
- to='ponos.task',
- ),
- ),
- migrations.AddConstraint(
- model_name='datasetelement',
- constraint=models.UniqueConstraint(
- fields=('dataset', 'element', 'set'),
- name='unique_dataset_elements',
- ),
- ),
- migrations.AddConstraint(
- model_name='dataset',
- constraint=models.UniqueConstraint(
- fields=('corpus', 'name'),
- name='unique_dataset_names',
- ),
- ),
- ]
diff --git a/arkindex/users/migrations/0001_initial.py b/arkindex/users/migrations/0001_initial.py
index f376013b9cc186767d0a97539de9f9e3414fb111..f09617397f666eddc783a58aa24b3385cf1e5eef 100644
--- a/arkindex/users/migrations/0001_initial.py
+++ b/arkindex/users/migrations/0001_initial.py
@@ -1,56 +1,74 @@
-# Generated by Django 2.2.9 on 2020-01-17 15:39
+# Generated by Django 4.1.7 on 2023-05-29 11:49
import uuid
+import django.core.validators
import django.db.models.deletion
import enumfields.fields
from django.conf import settings
+from django.contrib.postgres.operations import CreateCollation
from django.db import migrations, models
import arkindex.users.models
-def create_internal_group(apps, schema_editor):
- """
- Create the internal group, used for permission checking
-
- In the event of a migrations reset, this should be kept to make deployments simpler.
- """
- db_alias = schema_editor.connection.alias
- Group = apps.get_model('auth', 'Group')
- Group.objects.using(db_alias).create(id=settings.INTERNAL_GROUP_ID, name='Internal')
-
-
class Migration(migrations.Migration):
initial = True
dependencies = [
- ('auth', '0011_update_proxy_permissions'),
- ('documents', '0001_initial'),
+ ('documents', '0002_initial'),
+ ('contenttypes', '0002_remove_content_type_name'),
]
operations = [
+ CreateCollation(
+ 'case_insensitive',
+ provider='icu',
+ locale='und-u-ks-level2',
+ deterministic=False,
+ ),
migrations.CreateModel(
name='User',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('password', models.CharField(max_length=128, verbose_name='password')),
('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
- ('email', models.EmailField(max_length=255, unique=True, verbose_name='email address')),
+ ('email', models.EmailField(db_collation='case_insensitive', max_length=255, verbose_name='email address')),
+ ('display_name', models.CharField(max_length=120)),
+ ('transkribus_email', models.EmailField(blank=True, max_length=255, null=True)),
('is_active', models.BooleanField(default=True)),
('is_admin', models.BooleanField(default=False)),
('verified_email', models.BooleanField(default=False)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('selected_elements', models.ManyToManyField(related_name='selection_users', through='documents.Selection', to='documents.element')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Group',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=64)),
+ ('public', models.BooleanField(default=False)),
+ ('use_in_new_project', models.BooleanField(default=False)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Right',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('content_id', models.UUIDField(editable=False)),
+ ('level', models.PositiveIntegerField(help_text='Maximum privilege level.', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(100)])),
+ ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')),
+ ('group', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rights', to='users.group')),
+ ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rights', to=settings.AUTH_USER_MODEL)),
],
- options={
- 'abstract': False,
- },
),
migrations.CreateModel(
name='OAuthCredentials',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('provider_name', models.CharField(choices=[('GitLabOAuthProvider', 'GitLab')], max_length=50)),
('provider_url', models.URLField()),
('status', enumfields.fields.EnumField(default='created', enum=arkindex.users.models.OAuthStatus, max_length=10)),
('token', models.CharField(blank=True, max_length=64, null=True)),
@@ -59,32 +77,36 @@ class Migration(migrations.Migration):
('account_name', models.CharField(blank=True, max_length=100, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='credentials', to=settings.AUTH_USER_MODEL)),
],
+ options={
+ 'verbose_name': 'OAuth credentials',
+ 'verbose_name_plural': 'OAuth credentials',
+ },
),
migrations.CreateModel(
- name='CorpusRight',
+ name='UserScope',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('can_write', models.BooleanField(default=False)),
- ('can_admin', models.BooleanField(default=False)),
- ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='corpus_right', to='documents.Corpus')),
- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='corpus_right', to=settings.AUTH_USER_MODEL)),
+ ('scope', enumfields.fields.EnumField(enum=arkindex.users.models.Scope, max_length=50)),
+ ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_scopes', to=settings.AUTH_USER_MODEL)),
],
options={
- 'unique_together': {('user', 'corpus')},
+ 'unique_together': {('user', 'scope')},
},
),
- migrations.AddField(
- model_name='user',
- name='corpus',
- field=models.ManyToManyField(through='users.CorpusRight', to='documents.Corpus'),
+ migrations.AddConstraint(
+ model_name='right',
+ constraint=models.CheckConstraint(check=models.Q(models.Q(('group_id__isnull', False), ('user_id__isnull', True)), models.Q(('group_id__isnull', True), ('user_id__isnull', False)), _connector='OR'), name='user_xor_group'),
+ ),
+ migrations.AddConstraint(
+ model_name='right',
+ constraint=models.UniqueConstraint(condition=models.Q(('user__isnull', False)), fields=('user', 'content_id', 'content_type'), name='right_user_unique_target'),
+ ),
+ migrations.AddConstraint(
+ model_name='right',
+ constraint=models.UniqueConstraint(condition=models.Q(('group__isnull', False)), fields=('group', 'content_id', 'content_type'), name='right_group_unique_target'),
),
- migrations.AddField(
+ migrations.AddConstraint(
model_name='user',
- name='groups',
- field=models.ManyToManyField(blank=True, related_name='users', to='auth.Group'),
+ constraint=models.UniqueConstraint(fields=('email',), name='email_unique'),
),
- migrations.RunPython(
- create_internal_group,
- reverse_code=migrations.RunPython.noop,
- )
]
diff --git a/arkindex/users/migrations/0002_userscope.py b/arkindex/users/migrations/0002_userscope.py
deleted file mode 100644
index d5cd0afdfa2b4ae7b875357954d920503ca50c36..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0002_userscope.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Generated by Django 2.2.10 on 2020-04-30 10:56
-
-import django.db.models.deletion
-import enumfields.fields
-from django.conf import settings
-from django.db import migrations, models
-
-from arkindex.users.models import Scope
-
-
-def add_scopes(apps, schema_editor):
- db_alias = schema_editor.connection.alias
- User = apps.get_model('users', 'User')
- UserScope = apps.get_model('users', 'UserScope')
- UserScope.objects.using(db_alias).bulk_create([
- UserScope(user=user, scope=scope)
- for user in User.objects.using(db_alias).all()
- for scope in (Scope.CreateIIIFImage, Scope.UploadS3Image)
- ])
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0001_initial'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='UserScope',
- fields=[
- ('id', models.AutoField(
- auto_created=True,
- primary_key=True,
- serialize=False,
- verbose_name='ID',
- )),
- ('scope', enumfields.fields.EnumField(
- enum=Scope,
- max_length=50,
- )),
- ('user', models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- related_name='user_scopes',
- to=settings.AUTH_USER_MODEL,
- )),
- ],
- options={
- 'unique_together': {('user', 'scope')},
- },
- ),
- migrations.RunPython(
- add_scopes,
- reverse_code=migrations.RunPython.noop,
- )
- ]
diff --git a/arkindex/users/migrations/0003_user_selected_elements.py b/arkindex/users/migrations/0003_user_selected_elements.py
deleted file mode 100644
index c24e7cd79c627a7607901124dd58f360653b0a0c..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0003_user_selected_elements.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-24 14:39
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('documents', '0016_selection'),
- ('users', '0002_userscope'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='user',
- name='selected_elements',
- field=models.ManyToManyField(
- related_name='selection_users',
- through='documents.Selection',
- to='documents.Element',
- ),
- ),
- ]
diff --git a/arkindex/users/migrations/0004_internal_bool.py b/arkindex/users/migrations/0004_internal_bool.py
deleted file mode 100644
index d9b9495ee72537bf14f3053c06eca65ccef4b552..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0004_internal_bool.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# Generated by Django 2.2.13 on 2020-07-21 08:00
-
-from django.conf import settings
-from django.db import migrations, models
-
-
-def group_to_bool(apps, schema_editor):
- Group = apps.get_model('auth', 'Group')
- if not Group.objects.exists():
- # Empty database
- return
- internal_group = Group.objects.get(id=settings.INTERNAL_GROUP_ID)
- internal_group.users.all().update(is_internal=True)
- internal_group.delete()
-
-
-def bool_to_group(apps, schema_editor):
- Group = apps.get_model('auth', 'Group')
- User = apps.get_model('users', 'User')
- try:
- internal_group_id = settings.INTERNAL_GROUP_ID
- except AttributeError:
- internal_group_id = None
-
- assert isinstance(internal_group_id, int), 'settings.INTERNAL_GROUP_ID must be defined to roll users.0004 back'
-
- internal_group, _ = Group.objects.get_or_create(id=internal_group_id, defaults={'name': 'Internal'})
- internal_group.users.set(User.objects.filter(is_internal=True))
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0003_user_selected_elements'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='user',
- name='is_internal',
- field=models.BooleanField(default=False),
- ),
- migrations.RunPython(
- group_to_bool,
- reverse_code=bool_to_group,
- )
- ]
diff --git a/arkindex/users/migrations/0005_user_transkribus_email.py b/arkindex/users/migrations/0005_user_transkribus_email.py
deleted file mode 100644
index ae38739bc3f71c5162fbfb0a27756ad89ffd5057..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0005_user_transkribus_email.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.2.13 on 2020-08-05 07:24
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0004_internal_bool'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='user',
- name='transkribus_email',
- field=models.EmailField(blank=True, max_length=255, null=True),
- ),
- ]
diff --git a/arkindex/users/migrations/0006_update_group_and_membership.py b/arkindex/users/migrations/0006_update_group_and_membership.py
deleted file mode 100644
index 32e19fc087f1f1f69cfaf1e9245eb32ce4c5cefc..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0006_update_group_and_membership.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Generated by Django 3.1.3 on 2020-11-16 14:40
-
-import uuid
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0005_user_transkribus_email'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Group',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=64)),
- ('public', models.BooleanField(default=False)),
- ],
- ),
- migrations.RemoveField(
- model_name='user',
- name='groups',
- ),
- migrations.CreateModel(
- name='Membership',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('level', models.PositiveIntegerField(default=0, help_text='User privilege level.')),
- ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='memberships', to='users.group')),
- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='memberships', to=settings.AUTH_USER_MODEL)),
- ],
- ),
- migrations.AddField(
- model_name='group',
- name='users',
- field=models.ManyToManyField(related_name='groups', through='users.Membership', to=settings.AUTH_USER_MODEL),
- ),
- ]
diff --git a/arkindex/users/migrations/0007_user_display_name.py b/arkindex/users/migrations/0007_user_display_name.py
deleted file mode 100644
index f215bd45dae091cb51ea7b462260015da73b9942..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0007_user_display_name.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.1.3 on 2020-11-18 15:42
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0006_update_group_and_membership'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='user',
- name='display_name',
- field=models.CharField(default='default name', max_length=120),
- preserve_default=False,
- )
- ]
diff --git a/arkindex/users/migrations/0008_populate_user_display_name.py b/arkindex/users/migrations/0008_populate_user_display_name.py
deleted file mode 100644
index 187cc1596c26c922fff53e1f610360ed5d042c9b..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0008_populate_user_display_name.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from django.db import migrations
-
-
-def get_name_from_email(apps, schema_editor):
- User = apps.get_model('users', 'User')
- for user in User.objects.filter(display_name='default name'):
- try:
- user.display_name = user.email.split('@')[0].replace('.', ' ').title()
- except Exception:
- user.display_name = user.email
- user.save()
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0007_user_display_name'),
- ]
-
- operations = [
- migrations.RunPython(
- get_name_from_email,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/users/migrations/0009_membership_uuid_and_validation.py b/arkindex/users/migrations/0009_membership_uuid_and_validation.py
deleted file mode 100644
index a54857f9c07224f5be7b9fb1aedd743a1e5794c0..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0009_membership_uuid_and_validation.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Generated by Django 3.1.3 on 2020-11-24 13:30
-
-import uuid
-
-import django
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0008_populate_user_display_name'),
- ]
-
- operations = [
- # Drop membership table and re-create it
- migrations.DeleteModel(
- name='Membership',
- ),
- migrations.RemoveField(
- model_name='group',
- name='users',
- ),
- migrations.CreateModel(
- name='Membership',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('level', models.PositiveIntegerField(help_text='User privilege level.', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(100)])),
- ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='memberships', to='users.group')),
- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='memberships', to=settings.AUTH_USER_MODEL)),
- ],
- options={
- 'unique_together': {('user', 'group')},
- },
- ),
- migrations.AddField(
- model_name='group',
- name='users',
- field=models.ManyToManyField(related_name='groups', through='users.Membership', to=settings.AUTH_USER_MODEL),
- ),
- ]
diff --git a/arkindex/users/migrations/0010_right_gfk.py b/arkindex/users/migrations/0010_right_gfk.py
deleted file mode 100644
index 956364df2142a13ff4f45d02e43c89a1858884a0..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0010_right_gfk.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Generated by Django 3.1.3 on 2020-12-09 08:51
-
-import uuid
-
-import django.core.validators
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('contenttypes', '0002_remove_content_type_name'),
- ('users', '0009_membership_uuid_and_validation'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Right',
- fields=[
- ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
- ('content_id', models.UUIDField(editable=False)),
- ('level', models.PositiveIntegerField(help_text='Maximum privilege level.', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(100)])),
- ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')),
- ],
- ),
- migrations.RemoveField(
- model_name='group',
- name='users',
- ),
- migrations.DeleteModel(
- name='Membership',
- ),
- migrations.AddField(
- model_name='right',
- name='group',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rights', to='users.group'),
- ),
- migrations.AddField(
- model_name='right',
- name='user',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rights', to=settings.AUTH_USER_MODEL),
- ),
- migrations.AddConstraint(
- model_name='right',
- constraint=models.UniqueConstraint(fields=('user', 'group', 'content_id', 'content_type'), name='right_unique_target'),
- ),
- migrations.AddConstraint(
- model_name='right',
- constraint=models.CheckConstraint(check=models.Q(models.Q(('group_id__isnull', False), ('user_id__isnull', True)), models.Q(('group_id__isnull', True), ('user_id__isnull', False)), _connector='OR'), name='user_xor_group'),
- ),
- ]
diff --git a/arkindex/users/migrations/0011_corpus_rights.py b/arkindex/users/migrations/0011_corpus_rights.py
deleted file mode 100644
index 81397ab3dbea77f711f293d0d6db3782fea2f052..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0011_corpus_rights.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from django.contrib.contenttypes.models import ContentType
-from django.db import migrations
-
-import arkindex.users.models
-
-
-def create_rights(apps, schema_editor):
- CorpusRight = apps.get_model('users', 'CorpusRight')
- Right = apps.get_model('users', 'Right')
- Corpus = apps.get_model('documents', 'Corpus')
- corpus_type_id = ContentType.objects.get_for_model(Corpus).id
- # We should user the get_for_model method
- new_rights = []
- for corpus_right in CorpusRight.objects.all():
- RoleEnum = arkindex.users.models.Role
- role = RoleEnum.Guest
- if corpus_right.can_write:
- role = RoleEnum.Contributor
- if corpus_right.can_admin:
- role = RoleEnum.Admin
- new_rights.append(Right(
- user_id=corpus_right.user_id,
- content_id=corpus_right.corpus_id,
- content_type_id=corpus_type_id,
- level=role.value
- ))
- Right.objects.bulk_create(new_rights)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0010_right_gfk'),
- ]
-
- operations = [
- migrations.RunPython(
- create_rights,
- reverse_code=migrations.RunPython.noop,
- elidable=True
- )
- ]
diff --git a/arkindex/users/migrations/0012_drop_corpus_right.py b/arkindex/users/migrations/0012_drop_corpus_right.py
deleted file mode 100644
index 383d0945b5ffa5c312a7d457d98dc30444fb071a..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0012_drop_corpus_right.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Generated by Django 3.1.3 on 2020-12-18 14:58
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0011_corpus_rights'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='user',
- name='corpus',
- ),
- migrations.DeleteModel(
- name='CorpusRight',
- ),
- ]
diff --git a/arkindex/users/migrations/0013_user_created_updated_fields.py b/arkindex/users/migrations/0013_user_created_updated_fields.py
deleted file mode 100644
index 962bd31950eda2fb307ba9dba8bb64449b4f6152..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0013_user_created_updated_fields.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Generated by Django 3.1.7 on 2021-05-04 09:10
-
-import django.utils.timezone
-from django.contrib.contenttypes.models import ContentType
-from django.db import migrations, models
-
-from arkindex.users.models import Role
-
-
-def update_creation_date(apps, schema_editor):
- """
- Update User creation date to his "My Project" creation date (when the project is available)
- instead of default timezone.now
- """
- db_alias = schema_editor.connection.alias
- Corpus = apps.get_model('documents', 'Corpus')
- corpus_content_type = ContentType.objects.get_for_model(Corpus)
- Right = apps.get_model('users', 'Right')
- User = apps.get_model('users', 'User')
-
- to_update = []
- corpora = Corpus.objects.using(db_alias).filter(name='My Project')
- for corpus in corpora:
- try:
- admin = Right.objects.using(db_alias).filter(content_type=corpus_content_type.id, content_id=corpus.id).get(level__gte=Role.Admin.value, user__isnull=False).user
- admin.created = corpus.created
- to_update.append(admin)
- except (Right.DoesNotExist, Right.MultipleObjectsReturned):
- continue
- User.objects.using(db_alias).bulk_update(to_update, ['created'])
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0012_drop_corpus_right'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='user',
- name='created',
- field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
- preserve_default=False,
- ),
- migrations.AddField(
- model_name='user',
- name='updated',
- field=models.DateTimeField(auto_now=True),
- ),
- migrations.RunPython(
- update_creation_date,
- reverse_code=migrations.RunPython.noop,
- )
- ]
diff --git a/arkindex/users/migrations/0014_alter_oauthcredentials_options.py b/arkindex/users/migrations/0014_alter_oauthcredentials_options.py
deleted file mode 100644
index 08d287b173299fce8cc89c19ce5f59b837e14059..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0014_alter_oauthcredentials_options.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 3.2.6 on 2021-11-18 13:16
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0013_user_created_updated_fields'),
- ]
-
- operations = [
- migrations.AlterModelOptions(
- name='oauthcredentials',
- options={'verbose_name': 'OAuth credentials', 'verbose_name_plural': 'OAuth credentials'},
- ),
- ]
diff --git a/arkindex/users/migrations/0015_oauthcredentials_provider_choices.py b/arkindex/users/migrations/0015_oauthcredentials_provider_choices.py
deleted file mode 100644
index 086f36f1987045c29f80dc4a32c7115798a83e0c..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0015_oauthcredentials_provider_choices.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.6 on 2021-11-26 14:47
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0014_alter_oauthcredentials_options'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='oauthcredentials',
- name='provider_name',
- field=models.CharField(choices=[('gitlab', 'GitLab')], max_length=50),
- ),
- ]
diff --git a/arkindex/users/migrations/0016_alter_user_email_user_email_unique.py b/arkindex/users/migrations/0016_alter_user_email_user_email_unique.py
deleted file mode 100644
index a92635ff8773b791136df9b55e88d9e4bbe22ff0..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0016_alter_user_email_user_email_unique.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Generated by Django 4.0.2 on 2022-02-28 11:36
-
-from django.contrib.postgres.operations import CreateCollation
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0015_oauthcredentials_provider_choices'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='user',
- name='email',
- field=models.EmailField(max_length=255, verbose_name='email address'),
- ),
- migrations.AddConstraint(
- model_name='user',
- constraint=models.UniqueConstraint(fields=('email',), name='email_unique'),
- ),
- CreateCollation(
- 'case_insensitive',
- provider='icu',
- locale='und-u-ks-level2',
- deterministic=False,
- ),
- migrations.AlterField(
- model_name='user',
- name='email',
- field=models.EmailField(db_collation='case_insensitive', max_length=255, verbose_name='email address'),
- ),
- ]
diff --git a/arkindex/users/migrations/0017_right_unique_constraints.py b/arkindex/users/migrations/0017_right_unique_constraints.py
deleted file mode 100644
index 57b3276b592e77722e798c4cbe462c5740425d1e..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0017_right_unique_constraints.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Generated by Django 4.0.1 on 2022-04-08 13:50
-
-from django.db import migrations, models
-
-FORWARD_SQL = [
- """
- WITH duplicate_rights AS (
- SELECT id, ROW_NUMBER() OVER (
- PARTITION BY group_id, content_type_id, content_id
- ORDER BY level DESC
- ) AS number
- FROM users_right
- WHERE user_id IS NULL
- )
- DELETE FROM users_right
- USING duplicate_rights
- WHERE duplicate_rights.id = users_right.id
- AND duplicate_rights.number > 1
- """,
- """
- WITH duplicate_rights AS (
- SELECT id, ROW_NUMBER() OVER (
- PARTITION BY user_id, content_type_id, content_id
- ORDER BY level DESC
- ) AS number
- FROM users_right
- WHERE group_id IS NULL
- )
- DELETE FROM users_right
- USING duplicate_rights
- WHERE duplicate_rights.id = users_right.id
- AND duplicate_rights.number > 1
- """
-]
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0016_alter_user_email_user_email_unique'),
- ]
-
- operations = [
- migrations.RemoveConstraint(
- model_name='right',
- name='right_unique_target',
- ),
- migrations.RunSQL(
- sql=FORWARD_SQL,
- reverse_sql=migrations.RunSQL.noop,
- ),
- migrations.AddConstraint(
- model_name='right',
- constraint=models.UniqueConstraint(
- condition=models.Q(user__isnull=False),
- fields=('user', 'content_id', 'content_type'),
- name='right_user_unique_target',
- ),
- ),
- migrations.AddConstraint(
- model_name='right',
- constraint=models.UniqueConstraint(
- condition=models.Q(group__isnull=False),
- fields=('group', 'content_id', 'content_type'),
- name='right_group_unique_target',
- ),
- ),
- ]
diff --git a/arkindex/users/migrations/0018_remove_oauthcredentials_provider_name.py b/arkindex/users/migrations/0018_remove_oauthcredentials_provider_name.py
deleted file mode 100644
index d1797c60ef1890c9a3a496753a654879c390520b..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0018_remove_oauthcredentials_provider_name.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.0.7 on 2022-10-18 15:51
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0017_right_unique_constraints'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='oauthcredentials',
- name='provider_name',
- ),
- ]
diff --git a/arkindex/users/migrations/0019_group_use_in_new_project.py b/arkindex/users/migrations/0019_group_use_in_new_project.py
deleted file mode 100644
index 0d923844828a43b6a67d0251537c3bd176f56ef7..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0019_group_use_in_new_project.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.1.7 on 2023-03-13 12:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("users", "0018_remove_oauthcredentials_provider_name"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="group",
- name="use_in_new_project",
- field=models.BooleanField(default=False),
- ),
- ]
diff --git a/arkindex/users/migrations/0020_remove_user_is_internal.py b/arkindex/users/migrations/0020_remove_user_is_internal.py
deleted file mode 100644
index 65573ae5960a8941275d32974f1f7fe02e097b65..0000000000000000000000000000000000000000
--- a/arkindex/users/migrations/0020_remove_user_is_internal.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Generated by Django 4.1.7 on 2023-03-09 15:55
-
-from django.db import migrations
-
-
-def promote_internal_to_admin(apps, schema_editor):
- User = apps.get_model('users', 'User')
- if User.objects.filter(is_internal=True).exists():
- print('\nThe following internal users will be promoted to admins:')
- print('\n'.join(User.objects.filter(is_internal=True).values_list('email', flat=True)))
- User.objects.filter(is_internal=True).update(is_admin=True)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('users', '0019_group_use_in_new_project'),
- ]
-
- operations = [
- migrations.RunPython(
- promote_internal_to_admin,
- reverse_code=migrations.RunPython.noop,
- ),
- migrations.RemoveField(
- model_name='user',
- name='is_internal',
- ),
- ]