diff --git a/arkindex/process/api.py b/arkindex/process/api.py
index 386a1119103dbc93a293c89997dbbedd157e5f41..cd56db41de1ab8ad1f566d67997a46e14f68f8a9 100644
--- a/arkindex/process/api.py
+++ b/arkindex/process/api.py
@@ -920,7 +920,9 @@ class WorkerTypesList(ListAPIView):
description=dedent("""
Create a new version for a worker.
- The user must have an administrator access to the worker.
+ - The user must have an administrator access to the worker.
+ - If the worker has a `repository_url` set, then the version must be created with a `revision_url`.
+ - Worker versions created via the Ponos task authentication must have a `revision_url`.
""")
)
)
diff --git a/arkindex/process/managers.py b/arkindex/process/managers.py
index 3abe706ed374ad400d1b325e529d32d0d386d428..c7ca82b2d00d2fd1e91b162e86a5013a7543203a 100644
--- a/arkindex/process/managers.py
+++ b/arkindex/process/managers.py
@@ -193,7 +193,6 @@ class WorkerVersionManager(Manager):
self
# Required by WorkerRun.build_summary
.select_related("worker", "revision")
- .prefetch_related("revision__refs")
.get(id=settings.IMPORTS_WORKER_VERSION)
)
diff --git a/arkindex/process/migrations/0040_worker_repository_url_workerversion_revision_url_and_more.py b/arkindex/process/migrations/0040_worker_repository_url_workerversion_revision_url_and_more.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b6d09d77d2e51310174c48325c4025f7a78d3e7
--- /dev/null
+++ b/arkindex/process/migrations/0040_worker_repository_url_workerversion_revision_url_and_more.py
@@ -0,0 +1,82 @@
+# Generated by Django 5.0.6 on 2024-07-11 14:52
+
+from django.db import migrations, models
+from django.db.models import CharField, F, OuterRef, Subquery, Value
+from django.db.models.functions import Concat
+
+from arkindex.process.models import GitRefType
+from arkindex.project.tools import RTrimChr
+
+
+def update_git_refs(apps, schema):
+ Worker = apps.get_model("process", "Worker")
+ WorkerVersion = apps.get_model("process", "WorkerVersion")
+
+ Worker.objects.update(
+ repository_url=Subquery(
+ Worker.objects
+ .filter(pk=OuterRef("pk"))
+ .values("repository__url")
+ )
+ )
+
+ WorkerVersion.objects.update(
+ revision_url=Subquery(
+ WorkerVersion.objects
+ .filter(pk=OuterRef("pk"))
+ .annotate(
+ rev_url=Concat(
+ RTrimChr("worker__repository__url", Value("/")),
+ Value("/commit/"),
+ F("revision__hash"),
+ output_field=CharField(),
+ )
+ )
+ .values("rev_url")
+ ),
+ tag=Subquery(
+ WorkerVersion.objects
+ .filter(
+ pk=OuterRef("pk"),
+ revision__refs__type=GitRefType.Tag,
+ )
+ .order_by("-revision__refs__name")
+ [:1]
+ .values("revision__refs__name")
+ )
+ )
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("documents", "0012_alter_transcriptionentity_id"),
+ ("process", "0039_worker_configuration_name_not_empty"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="worker",
+ name="repository_url",
+ field=models.URLField(blank=True, default=None, max_length=250, null=True),
+ ),
+ migrations.AddField(
+ model_name="workerversion",
+ name="revision_url",
+ field=models.URLField(blank=True, default=None, max_length=250, null=True),
+ ),
+ migrations.AddField(
+ model_name="workerversion",
+ name="tag",
+ field=models.CharField(blank=True, default=None, max_length=50, null=True),
+ ),
+ migrations.AddConstraint(
+ model_name="workerversion",
+ constraint=models.UniqueConstraint(condition=models.Q(("tag__isnull", False)), fields=("worker", "tag"), name="workerversion_unique_tag"),
+ ),
+ migrations.RunPython(
+ update_git_refs,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ ),
+ ]
diff --git a/arkindex/process/models.py b/arkindex/process/models.py
index d9932105751a09889d8101c41d1d60731cb594e6..379de8af11d9dca7d2148f84cf905ca5cd2f9292 100644
--- a/arkindex/process/models.py
+++ b/arkindex/process/models.py
@@ -610,6 +610,8 @@ class Worker(models.Model):
archived = models.DateTimeField(null=True, blank=True, default=None)
+ repository_url = models.URLField(null=True, blank=True, max_length=250, default=None)
+
objects = WorkerManager()
class Meta:
@@ -758,6 +760,10 @@ class WorkerVersion(models.Model):
# For workers imported with `arkindex workers publish`, the tag of the Docker image
docker_image_iid = models.CharField(null=True, blank=True, max_length=512)
+ # URL of the commit for this version, when worker is based on a repository
+ revision_url = models.URLField(null=True, blank=True, max_length=250, default=None)
+ tag = models.CharField(blank=True, null=True, max_length=50, default=None)
+
corpora = models.ManyToManyField(
"documents.Corpus",
through="process.CorpusWorkerVersion",
@@ -794,6 +800,11 @@ class WorkerVersion(models.Model):
name="workerversion_unique_version",
condition=Q(version__isnull=False),
),
+ models.UniqueConstraint(
+ fields=["worker", "tag"],
+ name="workerversion_unique_tag",
+ condition=Q(tag__isnull=False),
+ ),
]
def __str__(self):
@@ -1002,14 +1013,17 @@ class WorkerRun(models.Model):
"""
summary_text = f"Worker {self.version.worker.name} @ "
- if self.version.version is not None:
- summary_text += f"version {self.version.version}"
- else:
- git_ref_names = self.version.revision.refs.values_list("name", flat=True)
- if len(git_ref_names) > 0:
- summary_text += ", ".join(git_ref_names)
+ if self.version.tag is not None:
+ summary_text += f"{self.version.tag} "
+ if self.version.revision is not None:
+ summary_text += f"({self.version.truncated_id})"
else:
+ summary_text += f"(version {self.version.version})"
+ else:
+ if self.version.revision is not None:
summary_text += self.version.truncated_id
+ else:
+ summary_text += f"version {self.version.version}"
if self.model_version:
summary_text += f" with model {self.model_version.model.name} @ {self.model_version.truncated_id}"
diff --git a/arkindex/process/serializers/workers.py b/arkindex/process/serializers/workers.py
index 46b64548cc41971616e00cfe7919e1d49cfd8b35..a0f7a348c33b76da68278ca664de2bb032fe539d 100644
--- a/arkindex/process/serializers/workers.py
+++ b/arkindex/process/serializers/workers.py
@@ -55,7 +55,7 @@ class WorkerSerializer(WorkerLightSerializer):
archived = ArchivedField(required=False)
class Meta(WorkerLightSerializer.Meta):
- fields = ("id", "name", "description", "type", "slug", "repository_id", "archived")
+ fields = ("id", "name", "description", "type", "slug", "repository_id", "repository_url", "archived")
read_only_fields = ("id", "repository_id")
extra_kwargs = {
"description": {
@@ -78,6 +78,7 @@ class WorkerCreateSerializer(WorkerSerializer):
If the WorkerType does not exist, it is created automatically.
"""))
+ repository_url = serializers.URLField(allow_null=True, required=False)
# Creating an archived worker makes no sense
archived = ArchivedField(read_only=True)
@@ -115,6 +116,7 @@ class WorkerCreateSerializer(WorkerSerializer):
slug=validated_data["slug"],
name=validated_data["name"],
type=worker_type,
+ repository_url=validated_data.get("repository_url")
)
# Automatically grant an admin access to the creator of a local worker
worker.memberships.create(user=request.user, level=Role.Admin.value)
@@ -231,10 +233,12 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
state = EnumField(WorkerVersionState, required=False)
worker = WorkerLightSerializer(read_only=True)
revision = RevisionWithRefsSerializer(required=False, read_only=True, allow_null=True)
+ revision_url = serializers.URLField(required=False, read_only=True, allow_null=True)
gpu_usage = EnumField(FeatureUsage, required=False, default=FeatureUsage.Disabled)
model_usage = EnumField(FeatureUsage, required=False, default=FeatureUsage.Disabled)
# Ensure worker version configuration JSON body is an object
configuration = serializers.DictField()
+ tag = serializers.CharField(allow_null=True, max_length=50, required=False)
# Serialize worker with its basic information
class Meta:
@@ -243,6 +247,7 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
"id",
"configuration",
"revision",
+ "revision_url",
"version",
"docker_image_iid",
"state",
@@ -250,13 +255,21 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
"model_usage",
"worker",
"created",
+ "tag",
)
- read_only_fields = ("revision", "version")
+ read_only_fields = ("revision", "revision_url", "version")
# Avoid loading all revisions and all Ponos artifacts when opening this endpoint in a browser
extra_kwargs = {
"revision": {"style": {"base_template": "input.html"}},
}
+ def validate_tag(self, tag):
+ worker = self.instance.worker if self.instance else self.context["worker"]
+ existing_version = worker.versions.filter(tag=tag).exists()
+ if existing_version:
+ raise ValidationError("A version already exists for this worker with this tag.")
+ return tag
+
def validate_configuration(self, configuration):
errors = defaultdict(list)
user_configuration = configuration.get("user_configuration")
@@ -290,46 +303,11 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
class WorkerVersionCreateSerializer(WorkerVersionSerializer):
- revision_id = serializers.PrimaryKeyRelatedField(
+ revision_url = serializers.URLField(
required=False,
- write_only=True,
- queryset=Revision.objects.all(),
- allow_null=True,
- style={"base_template": "input.html"},
- help_text=dedent("""
- Git revision for this version.
-
- This field is required on workers linked to a repository.
- On other workers, it cannot be set as an automatic version number is attributed.
- """).strip(),
- source="revision"
+ allow_null=True
)
- class Meta (WorkerVersionSerializer.Meta):
- fields = WorkerVersionSerializer.Meta.fields + (
- "revision_id",
- )
-
- def validate_revision_id(self, revision):
- worker = self.context["worker"]
- if worker.versions.using("default").filter(revision=revision).exists():
- raise ValidationError("A version of this worker already exists with this revision")
- if isinstance(self.context["request"].auth, Task):
- if worker.repository_id is None:
- # Task authentication is forbidden on workers not linked to a repository
- raise ValidationError(
- "Task authentication requires to create a version on workers linked to a repository."
- )
- elif worker.repository_id is not None:
- raise ValidationError(
- "Ponos authentication is required to create a version on a worker linked to a repository."
- )
- if worker.repository is None:
- raise ValidationError("A revision cannot be set on a worker that is not linked to a repository.")
- if worker.repository_id != revision.repo_id:
- raise ValidationError("The revision must be part of the same repository as the worker.")
- return revision
-
def validate_docker_image_iid(self, docker_image_iid):
repository, _ = parse_repository_tag(docker_image_iid)
try:
@@ -346,19 +324,19 @@ class WorkerVersionCreateSerializer(WorkerVersionSerializer):
if worker.archived:
errors["worker"].append("This worker is archived.")
- revision = data.get("revision")
- if revision is None and worker.repository_id is not None:
- errors["revision_id"].append("A revision must be set on a worker that is linked to a repository.")
+ revision_url = data.get("revision_url")
+ if isinstance(self.context["request"].auth, Task) and not revision_url:
+ errors["revision_url"].append("This field is required when creating a version through Task authentication.")
+ if worker.repository_url and not revision_url:
+ errors["revision_url"].append("A revision url is required when creating a version for a worker linked to a repository.")
if errors:
raise ValidationError(errors)
- # Define a version number on workers that are not linked to a repository
- version = None
- if revision is None:
- last_version = worker.versions.using("default").aggregate(last_version=Max("version"))["last_version"]
- version = last_version + 1 if last_version else 1
- data["version"] = version
+ # Define a version number
+ last_version = worker.versions.using("default").aggregate(last_version=Max("version"))["last_version"]
+ data["version"] = last_version + 1 if last_version else 1
+
data["worker_id"] = worker.id
return data
diff --git a/arkindex/process/tests/test_corpus_worker_runs.py b/arkindex/process/tests/test_corpus_worker_runs.py
index 7426372bb790e351ee655448d77d4048d04886b4..fa5d1f8184ea9044ac0cae9ddb26bf699a886550 100644
--- a/arkindex/process/tests/test_corpus_worker_runs.py
+++ b/arkindex/process/tests/test_corpus_worker_runs.py
@@ -126,8 +126,10 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
"id": str(self.local_worker_version.id),
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"state": "created",
"version": 1,
+ "tag": None,
"worker": {
"id": str(self.local_worker_version.worker.id),
"name": "Custom worker",
@@ -146,7 +148,9 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.dla_worker_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
diff --git a/arkindex/process/tests/test_docker_worker_version.py b/arkindex/process/tests/test_docker_worker_version.py
index 97831efbe1e5d2cc86fd5ac59f8c36e5d9a3b0db..08ea29e1fa995188bdbe739902b26e63df03dfcd 100644
--- a/arkindex/process/tests/test_docker_worker_version.py
+++ b/arkindex/process/tests/test_docker_worker_version.py
@@ -267,7 +267,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"message": "created from docker image",
"refs": []
},
+ "revision_url": None,
"version": None,
+ "tag": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -332,7 +334,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
for ref in refs
],
},
+ "revision_url": None,
"version": None,
+ "tag": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -403,7 +407,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
for ref in refs
],
},
+ "revision_url": None,
"version": None,
+ "tag": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -499,7 +505,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"hash": "new_revision_hash",
"message": "Bruce was very clever",
},
+ "revision_url": None,
"version": None,
+ "tag": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -572,7 +580,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
},
],
},
+ "revision_url": None,
"version": None,
+ "tag": None,
"created": self.version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -629,7 +639,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"refs": [{"id": str(new_ref.id), "name": new_ref.name, "type": new_ref.type.value}],
},
"state": "available",
+ "revision_url": None,
"version": None,
+ "tag": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"worker": {
"id": str(new_version.worker.id),
diff --git a/arkindex/process/tests/test_user_workerruns.py b/arkindex/process/tests/test_user_workerruns.py
index 2b14a8300f4310c6eadade9e5796305607647334..4a76ae878d299d830502ce595560fdc321387c6a 100644
--- a/arkindex/process/tests/test_user_workerruns.py
+++ b/arkindex/process/tests/test_user_workerruns.py
@@ -101,8 +101,10 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"id": str(self.version_1.id),
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"state": "available",
"version": 1,
+ "tag": None,
"worker": {
"id": str(self.version_1.worker.id),
"name": "Recognizer",
@@ -137,8 +139,10 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"id": str(self.custom_version.id),
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"state": "created",
"version": 1,
+ "tag": None,
"worker": {
"id": str(self.custom_version.worker.id),
"name": "Custom worker",
@@ -218,8 +222,10 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"id": str(self.other_version.id),
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"state": "created",
"version": 2,
+ "tag": None,
"worker": {
"id": str(self.other_version.worker.id),
"name": "Custom worker",
@@ -433,8 +439,10 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"id": str(self.other_version.id),
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"state": "created",
"version": 2,
+ "tag": None,
"worker": {
"id": str(self.other_version.worker.id),
"name": "Custom worker",
diff --git a/arkindex/process/tests/test_workerruns.py b/arkindex/process/tests/test_workerruns.py
index 5ea79b7ac59bf29a2bd1633a1469764469ce95b4..47387dac7e63e0c2ffa3929b1813aa0187dabf7a 100644
--- a/arkindex/process/tests/test_workerruns.py
+++ b/arkindex/process/tests/test_workerruns.py
@@ -136,7 +136,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -404,7 +406,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -476,7 +480,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -524,94 +530,79 @@ class TestWorkerRuns(FixtureAPITestCase):
self.assertDictEqual(response.json(), {"configuration_id": ["The configuration must be part of the same worker."]})
- def test_list_summary_with_git_refs(self):
- """
- Create a worker run with a worker version related to a revision tagged with a GitRef
- """
+ def test_summary(self):
self.client.force_login(self.user)
-
repo = Repository.objects.create(url="http://nerv.co.jp/nerv/eva/")
- # Version with no git refs
revision = Revision.objects.create(
- repo_id=repo.id,
- hash=uuid.uuid4().hex,
- message="Fake revision",
- author="Teklia Bot"
+ repo=repo,
+ hash="1",
+ message="commit message",
+ author="bob",
)
- git_ref_names = ["main", "develop", "trunk", "master", "patate", "pouet"]
- expected_refs = []
- for git_ref_name in git_ref_names:
- new_ref = revision.refs.create(
- name=git_ref_name,
- type=GitRefType.Branch,
- repository=revision.repo
- )
- expected_refs.append({"id": str(new_ref.id), "name": git_ref_name, "type": "branch"})
-
- version = WorkerVersion.objects.create(
- worker=self.worker_1,
- revision=revision,
- configuration={},
+ test_version = self.worker_1.versions.create(
+ version=2,
state=WorkerVersionState.Available,
- docker_image_iid=self.version_1.docker_image_iid,
+ docker_image_iid="registry.gitlab.com/dead-sea-scrolls:004",
+ model_usage=FeatureUsage.Supported
)
- with self.assertNumQueries(8):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process_2.id)}),
- data={"worker_version_id": str(version.id), "parents": []},
- format="json",
- )
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+ cases = [
+ ("eva-01", revision, self.model_version_1, self.configuration_1, f"Worker Recognizer @ eva-01 ({str(test_version.id)[:6]}) with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'"),
+ (None, revision, self.model_version_1, self.configuration_1, f"Worker Recognizer @ {str(test_version.id)[:6]} with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'"),
+ ("eva-01", revision, None, self.configuration_1, f"Worker Recognizer @ eva-01 ({str(test_version.id)[:6]}) using configuration 'My config'"),
+ (None, revision, self.model_version_1, None, f"Worker Recognizer @ {str(test_version.id)[:6]} with model My model @ {str(self.model_version_1.id)[:6]}"),
+ ("eva-01", revision, None, None, f"Worker Recognizer @ eva-01 ({str(test_version.id)[:6]})"),
+ (None, revision, None, None, f"Worker Recognizer @ {str(test_version.id)[:6]}"),
+ ("eva-01", None, self.model_version_1, self.configuration_1, f"Worker Recognizer @ eva-01 (version 2) with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'"),
+ (None, None, self.model_version_1, self.configuration_1, f"Worker Recognizer @ version 2 with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'"),
+ ("eva-01", None, None, self.configuration_1, "Worker Recognizer @ eva-01 (version 2) using configuration 'My config'"),
+ (None, None, self.model_version_1, None, f"Worker Recognizer @ version 2 with model My model @ {str(self.model_version_1.id)[:6]}"),
+ ("eva-01", None, None, None, "Worker Recognizer @ eva-01 (version 2)"),
+ (None, None, None, None, "Worker Recognizer @ version 2"),
+ ]
- data = response.json()
- pk = data.pop("id")
- self.assertNotEqual(pk, self.run_1.id)
- self.assertDictEqual(data, {
- "worker_version": {
- "id": str(version.id),
- "configuration": {},
- "docker_image_iid": version.docker_image_iid,
- "gpu_usage": "disabled",
- "model_usage": FeatureUsage.Disabled.value,
- "revision": {
- "id": str(version.revision.id),
- "author": "Teklia Bot",
- "commit_url": f"http://nerv.co.jp/nerv/eva/commit/{version.revision.hash}",
- "created": version.revision.created.isoformat().replace("+00:00", "Z"),
- "hash": version.revision.hash,
- "message": "Fake revision",
- "refs": expected_refs,
- },
- "version": None,
- "created": version.created.isoformat().replace("+00:00", "Z"),
- "state": "available",
- "worker": {
- "id": str(self.worker_1.id),
- "name": "Recognizer",
- "slug": "reco",
- "type": "recognizer"
+ for tag, revision, model_version, config, expected_summary in cases:
+ with self.subTest(tag=tag, revision=revision, model_version=model_version, config=config):
+ # Clear the process of worker runs
+ self.process_2.worker_runs.all().delete()
+
+ num_queries = 6
+
+ test_version.tag = tag
+ test_version.revision = None
+ test_version.version = 2
+ if revision:
+ test_version.revision = revision
+ test_version.version = None
+ # If there is a revision, it adds a query
+ num_queries += 1
+ test_version.save()
+
+ payload = {
+ "worker_version_id": str(test_version.id),
+ "parents": [],
}
- },
- "parents": [],
- "model_version": None,
- "configuration": None,
- "process": {
- "id": str(self.process_2.id),
- "activity_state": "disabled",
- "corpus": str(self.corpus.id),
- "chunks": 1,
- "mode": "workers",
- "name": None,
- "state": "unscheduled",
- "use_cache": False,
- },
- "summary": "Worker Recognizer @ main, develop, trunk, master, patate, pouet",
- "use_gpu": False,
- })
- run = WorkerRun.objects.get(pk=pk)
- # Check generated summary
- self.assertEqual(run.summary, "Worker Recognizer @ main, develop, trunk, master, patate, pouet")
+ if model_version:
+ payload["model_version_id"] = model_version.id
+ # If there is a model version, it adds a query
+ num_queries += 1
+ if config:
+ payload["configuration_id"] = config.id
+ # If there is a configuration, it adds a query
+ num_queries += 1
+
+ with self.assertNumQueries(num_queries):
+ response = self.client.post(
+ reverse("api:worker-run-list", kwargs={"pk": str(self.process_2.id)}),
+ data=payload,
+ format="json",
+ )
+ if response.status_code == 400:
+ self.assertDictEqual(response.json(), {})
+ self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+ created_run = WorkerRun.objects.get(pk=response.json()["id"])
+ self.assertEqual(created_run.summary, expected_summary)
def test_retrieve_requires_login(self):
with self.assertNumQueries(0):
@@ -657,7 +648,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -713,7 +706,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -798,7 +793,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "created",
"worker": {
@@ -913,7 +910,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -980,7 +979,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -1217,7 +1218,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -1275,7 +1278,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -1341,7 +1346,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -1708,7 +1715,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Supported.value,
"revision": None,
+ "revision_url": None,
"version": version_with_model.version,
+ "tag": None,
"created": version_with_model.created.isoformat().replace("+00:00", "Z"),
"state": "created",
"worker": {
@@ -1798,7 +1807,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Required.value,
"revision": None,
+ "revision_url": None,
"version": version_with_model.version,
+ "tag": None,
"created": version_with_model.created.isoformat().replace("+00:00", "Z"),
"state": "created",
"worker": {
@@ -1864,7 +1875,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -2082,7 +2095,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -2140,7 +2155,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -2203,7 +2220,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -2557,7 +2576,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Required.value,
"revision": None,
+ "revision_url": None,
"version": version_with_model.version,
+ "tag": None,
"created": version_with_model.created.isoformat().replace("+00:00", "Z"),
"state": "created",
"worker": {
@@ -2644,7 +2665,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Required.value,
"revision": None,
+ "revision_url": None,
"version": version_with_model.version,
+ "tag": None,
"created": version_with_model.created.isoformat().replace("+00:00", "Z"),
"state": "created",
"worker": {
@@ -2709,7 +2732,9 @@ class TestWorkerRuns(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
diff --git a/arkindex/process/tests/test_workerruns_use_gpu.py b/arkindex/process/tests/test_workerruns_use_gpu.py
index 5dbb690e864bc0dc98f67b79169abd98a9659da0..cc9e66b7de594fd578885b1d8eb37d4601f4c0a5 100644
--- a/arkindex/process/tests/test_workerruns_use_gpu.py
+++ b/arkindex/process/tests/test_workerruns_use_gpu.py
@@ -77,7 +77,9 @@ class TestWorkerRunsGPU(FixtureAPITestCase):
"gpu_usage": worker_version.gpu_usage.value,
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": worker_version.version,
+ "tag": None,
"created": worker_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -129,7 +131,9 @@ class TestWorkerRunsGPU(FixtureAPITestCase):
"gpu_usage": FeatureUsage.Required.value,
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 2,
+ "tag": None,
"created": self.version_gpu_required.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
@@ -222,7 +226,9 @@ class TestWorkerRunsGPU(FixtureAPITestCase):
"gpu_usage": worker_version.gpu_usage.value,
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": worker_version.version,
+ "tag": None,
"created": worker_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
diff --git a/arkindex/process/tests/test_workers.py b/arkindex/process/tests/test_workers.py
index a298519cefde678bf23b7bfc9980f1d5dee0cb11..d68b4d5939e7b3b9d48dcc9f87b6c929f80e4b55 100644
--- a/arkindex/process/tests/test_workers.py
+++ b/arkindex/process/tests/test_workers.py
@@ -12,7 +12,6 @@ from arkindex.process.models import (
GitRefType,
ProcessMode,
Repository,
- Revision,
Worker,
WorkerConfiguration,
WorkerType,
@@ -93,6 +92,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_custom.id),
"repository_id": None,
+ "repository_url": None,
"name": "Custom worker",
"description": "",
"slug": "custom",
@@ -102,6 +102,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_dla.id),
"repository_id": None,
+ "repository_url": None,
"name": "Document layout analyser",
"description": "",
"slug": "dla",
@@ -111,6 +112,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.init_worker.id),
"repository_id": None,
+ "repository_url": None,
"name": "Elements Initialisation Worker",
"description": "",
"slug": "initialisation",
@@ -120,6 +122,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_file_import.id),
"repository_id": None,
+ "repository_url": None,
"name": "File import",
"description": "",
"slug": "file_import",
@@ -129,6 +132,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_generic.id),
"repository_id": None,
+ "repository_url": None,
"name": "Generic worker with a Model",
"description": "",
"slug": "generic",
@@ -138,6 +142,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_reco.id),
"repository_id": None,
+ "repository_url": None,
"name": "Recognizer",
"description": "",
"slug": "reco",
@@ -147,6 +152,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_gpu.id),
"repository_id": None,
+ "repository_url": None,
"name": "Worker requiring a GPU",
"description": "",
"slug": "worker-gpu",
@@ -180,6 +186,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_generic.id),
"repository_id": None,
+ "repository_url": None,
"name": "Generic worker with a Model",
"description": "",
"slug": "generic",
@@ -209,6 +216,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"results": [{
"id": str(self.worker_reco.id),
"repository_id": None,
+ "repository_url": None,
"name": "Recognizer",
"description": "",
"slug": "reco",
@@ -280,6 +288,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_dla.id),
"repository_id": None,
+ "repository_url": None,
"name": "Document layout analyser",
"description": "",
"slug": "dla",
@@ -307,6 +316,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_dla.id),
"repository_id": None,
+ "repository_url": None,
"name": "Document layout analyser",
"description": "",
"slug": "dla",
@@ -335,6 +345,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_dla.id),
"repository_id": None,
+ "repository_url": None,
"name": "Document layout analyser",
"description": "",
"slug": "dla",
@@ -375,6 +386,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_dla.id),
"repository_id": None,
+ "repository_url": None,
"name": "Document layout analyser",
"description": "",
"slug": "dla",
@@ -426,6 +438,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(self.worker_dla.id),
"repository_id": None,
+ "repository_url": None,
"name": "Document layout analyser",
"description": "",
"slug": "dla",
@@ -461,6 +474,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
{
"id": str(worker_2.id),
"repository_id": str(repo2.id),
+ "repository_url": None,
"name": "Worker 2",
"description": "",
"slug": "worker_2",
@@ -484,6 +498,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
self.assertDictEqual(response.json(), {
"id": str(self.worker_custom.id),
"repository_id": None,
+ "repository_url": None,
"name": "Custom worker",
"description": "",
"slug": "custom",
@@ -551,7 +566,8 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
data={
"name": "Worker post",
"slug": "worker_post",
- "type": self.worker_type_dla.slug
+ "type": self.worker_type_dla.slug,
+ "repository_url": "https://gitlab.com/NERV/eva/"
}
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
@@ -568,6 +584,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"name": "Worker post",
"description": "",
"repository_id": None,
+ "repository_url": "https://gitlab.com/NERV/eva/",
"slug": "worker_post",
"type": "dla",
"archived": False,
@@ -732,6 +749,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"description": "New description",
"type": "dla",
"archived": False,
+ "repository_url": "https://gitlab.com/NERV/eva"
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -743,6 +761,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"description": "New description",
"type": "dla",
"repository_id": None,
+ "repository_url": "https://gitlab.com/NERV/eva",
"archived": False,
})
@@ -752,6 +771,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
self.assertEqual(self.worker_reco.description, "New description")
self.assertEqual(self.worker_reco.type, self.worker_type_dla)
self.assertEqual(self.worker_reco.repository, None)
+ self.assertEqual(self.worker_reco.repository_url, "https://gitlab.com/NERV/eva")
self.assertListEqual(filter_rights_mock.call_args_list, [
call(self.user, Worker, Role.Contributor.value),
])
@@ -846,6 +866,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"description": "New description",
"type": "dla",
"repository_id": None,
+ "repository_url": None,
"archived": new_value,
})
@@ -925,6 +946,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"description": "New description",
"type": "dla",
"repository_id": None,
+ "repository_url": None,
"archived": False,
})
@@ -1017,6 +1039,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"description": "",
"type": "recognizer",
"repository_id": None,
+ "repository_url": None,
"archived": new_value,
})
@@ -1032,6 +1055,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
version=2,
configuration={"a": "bc"},
state=WorkerVersionState.Error,
+ revision_url="https://gitlab.com/NERV/eva/commit/63e377e7f88c743d8428fc4e4eaedfc1c9356754"
)
last_version.created = "2050-09-09T09:09:09.090909Z"
last_version.save()
@@ -1060,7 +1084,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"slug": self.worker_reco.slug,
},
"version": 2,
+ "tag": None,
"revision": None,
+ "revision_url": "https://gitlab.com/NERV/eva/commit/63e377e7f88c743d8428fc4e4eaedfc1c9356754",
"created": "2050-09-09T09:09:09.090909Z",
},
{
@@ -1077,7 +1103,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"slug": self.worker_reco.slug,
},
"version": 1,
+ "tag": None,
"revision": None,
+ "revision_url": None,
"created": self.version_1.created.isoformat().replace("+00:00", "Z"),
}
]
@@ -1248,30 +1276,11 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
]
})
- def test_create_version_return_existing_revision(self):
- rev = self.repo.revisions.create(hash="001", message="something", author="someone")
- self.version_1.revision_id = rev.id
- self.version_1.version = None
- self.version_1.save()
- # A worker version already exists for this worker and this revision
- with self.assertNumQueries(4):
- response = self.client.post(
- reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
- data={"revision_id": str(rev.id), "configuration": {"test": "test1"}},
- format="json",
- HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertDictEqual(response.json(), {
- "revision_id": ["A version of this worker already exists with this revision"]
- })
-
def test_create_version_empty(self):
- with self.assertNumQueries(2):
+ self.client.force_login(self.user)
+ with self.assertNumQueries(3):
response = self.client.post(
- reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
- HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)})
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
@@ -1286,6 +1295,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
data={
"configuration": {},
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01"
},
format="json",
HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
@@ -1296,57 +1306,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"worker": ["This worker is archived."],
})
- def test_create_version_unrelated_revision(self):
- """
- It is not possible to create a version between a worker and a revision of two different repositories
- """
- self.worker_reco.repository = self.repo
+ def test_create_version_null_revision_url_requires_null_worker_repository_url(self):
+ self.worker_reco.repository_url = "https://gitlab.com/NERV/eva"
self.worker_reco.save()
- rev = Revision.objects.create(
- hash="1337",
- message="A worker import I am admin on",
- author="Yann",
- repo=Repository.objects.create(url="http://gitlab/repo")
- )
-
- with self.assertNumQueries(4):
- response = self.client.post(
- reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
- data={"revision_id": str(rev.id), "configuration": {"test": "test2"}},
- format="json",
- HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertDictEqual(response.json(), {
- "revision_id": ["The revision must be part of the same repository as the worker."]
- })
-
- def test_create_version_revision_requires_worker_repository(self):
- self.worker_custom.memberships.filter(user=self.user).update(level=Role.Admin.value)
- self.client.force_login(self.user)
- rev = self.repo.revisions.create(
- hash="1337",
- message="A worker import I am admin on",
- author="Yann",
- )
-
- with self.assertNumQueries(5):
- response = self.client.post(
- reverse("api:worker-versions", kwargs={"pk": str(self.worker_custom.id)}),
- data={"revision_id": str(rev.id), "configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value},
- format="json",
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertDictEqual(response.json(), {
- "revision_id": ["A revision cannot be set on a worker that is not linked to a repository."]
- })
-
- def test_create_version_null_revision_requires_null_worker_repository(self):
- self.worker_reco.repository = self.repo
- self.worker_reco.save()
- self.worker_custom.memberships.filter(user=self.user).update(level=Role.Admin.value)
self.client.force_login(self.user)
with self.assertNumQueries(3):
@@ -1358,31 +1320,24 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(response.json(), {
- "revision_id": ["A revision must be set on a worker that is linked to a repository."],
+ "revision_url": ["A revision url is required when creating a version for a worker linked to a repository."],
})
- def test_create_version_null_revision_forbidden_task_auth(self):
+ def test_create_version_null_revision_url_forbidden_task_auth(self):
"""
Ponos Task auth cannot create a version on a worker that is not linked to a repository.
"""
- self.worker_custom.versions.create(version=41, configuration={})
- rev = self.repo.revisions.create(
- hash="1337",
- message="A worker import I am admin on",
- author="Yann",
- )
-
- with self.assertNumQueries(4):
+ with self.assertNumQueries(2):
response = self.client.post(
reverse("api:worker-versions", kwargs={"pk": str(self.worker_custom.id)}),
- data={"revision_id": str(rev.id), "configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value},
+ data={"configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value},
format="json",
HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(response.json(), {
- "revision_id": ["Task authentication requires to create a version on workers linked to a repository."]
+ "revision_url": ["This field is required when creating a version through Task authentication."]
})
@patch("arkindex.users.utils.get_max_level", return_value=Role.Contributor.value)
@@ -1402,30 +1357,44 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
self.assertEqual(max_level_mock.call_count, 1)
self.assertEqual(max_level_mock.call_args, call(self.user, self.worker_custom))
- def test_create_version_user_auth_requires_null_repository(self):
+ def test_create_version_user_auth_with_worker_repository_url_ok(self):
self.client.force_login(self.user)
- self.worker_dla.repository = self.repo
+ self.worker_dla.repository_url = "https://gitlab.com/NERV/eva"
self.worker_dla.save()
- rev = self.repo.revisions.create(
- hash="1337",
- message="A worker import I am admin on",
- author="Yann",
- )
- with self.assertNumQueries(5):
+
+ with self.assertNumQueries(7):
response = self.client.post(
reverse("api:worker-versions", kwargs={"pk": str(self.worker_dla.id)}),
- data={"revision_id": str(rev.id), "configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value},
+ data={"revision_url": "https://gitlab.com/NERV/eva/commit/eva-01", "configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value},
format="json",
)
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+ data = response.json()
+ new_version = self.worker_dla.versions.get(id=data["id"])
self.assertDictEqual(response.json(), {
- "revision_id": ["Ponos authentication is required to create a version on a worker linked to a repository."]
+ "id": str(new_version.id),
+ "configuration": {"test": "test2"},
+ "created": new_version.created.isoformat().replace("+00:00", "Z"),
+ "docker_image_iid": None,
+ "gpu_usage": FeatureUsage.Disabled.value,
+ "model_usage": FeatureUsage.Required.value,
+ "revision": None,
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
+ "state": WorkerVersionState.Created.value,
+ "version": 2,
+ "tag": None,
+ "worker": {
+ "id": str(self.worker_dla.id),
+ "name": "Document layout analyser",
+ "slug": "dla",
+ "type": "dla",
+ },
})
- def test_create_version_null_revision(self):
+ def test_create_version_null_revision_url(self):
"""
- A worker version can be created with no revision on a worker that has no repository.
- Its version number is automatically incremented.
+ A worker version can be created with no revision_url through user authentication
"""
self.worker_custom.memberships.filter(user=self.user).update(level=Role.Admin.value)
self.worker_custom.versions.create(version=41, configuration={})
@@ -1447,8 +1416,10 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"gpu_usage": FeatureUsage.Disabled.value,
"model_usage": FeatureUsage.Required.value,
"revision": None,
+ "revision_url": None,
"state": WorkerVersionState.Created.value,
"version": 42,
+ "tag": None,
"worker": {
"id": str(self.worker_custom.id),
"name": "Custom worker",
@@ -1474,7 +1445,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
def test_create_version(self):
response = self.client.post(
reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
- data={"configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value},
+ data={"configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value, "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01"},
format="json",
HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
)
@@ -1488,10 +1459,54 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
self.assertEqual(data["gpu_usage"], "disabled")
self.assertEqual(data["model_usage"], FeatureUsage.Required.value)
+ def test_create_version_with_tag(self):
+ with self.assertNumQueries(7):
+ response = self.client.post(
+ reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
+ data={"configuration": {"test": "test2"}, "model_usage": FeatureUsage.Required.value, "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01", "tag": "eva-01"},
+ format="json",
+ HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ )
+ self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+ data = response.json()
+ self.assertNotEqual(data["id"], str(self.version_1.id))
+ self.assertEqual(data["configuration"], {"test": "test2"})
+ self.assertEqual(data["version"], 2)
+ self.assertEqual(data["state"], "created")
+ self.assertEqual(data["gpu_usage"], "disabled")
+ self.assertEqual(data["model_usage"], FeatureUsage.Required.value)
+ self.assertEqual(data["tag"], "eva-01")
+
+ def test_create_version_unique_tag(self):
+ self.version_1.tag = "eva-01"
+ self.version_1.save()
+
+ with self.assertNumQueries(3):
+ response = self.client.post(
+ reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
+ data={"configuration": {"test": "test2"}, "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01", "tag": "eva-01"},
+ format="json",
+ HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ )
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertDictEqual(response.json(), {"tag": ["A version already exists for this worker with this tag."]})
+
+ def test_create_version_empty_tag(self):
+ with self.assertNumQueries(2):
+ response = self.client.post(
+ reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
+ data={"configuration": {"test": "test2"}, "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01", "tag": ""},
+ format="json",
+ HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ )
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertDictEqual(response.json(), {"tag": ["This field may not be blank."]})
+
def test_create_version_wrong_gpu_usage(self):
response = self.client.post(
reverse("api:worker-versions", kwargs={"pk": str(self.worker_reco.id)}),
- data={"configuration": {"test": "test2"}, "gpu_usage": "not_supported"},
+ data={"configuration": {"test": "test2"}, "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01", "gpu_usage": "not_supported"},
format="json",
HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
)
@@ -1502,6 +1517,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
reverse("api:worker-versions", kwargs={"pk": str(self.worker_dla.id)}),
data={
"configuration": {"beep": "boop"},
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
"gpu_usage": "disabled",
},
format="json",
@@ -1520,6 +1536,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
}
},
"gpu_usage": "disabled",
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
},
format="json",
HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
@@ -1551,6 +1568,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"demo_dict": {"title": "Demo Dict", "type": "dict", "required": True, "default": {"a": "b", "c": "d"}},
}
},
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
"gpu_usage": "disabled",
},
format="json",
@@ -1593,6 +1611,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"demo_choice": {"title": "Decisions", "type": "enum", "required": True, "default": 1, "choices": [1, 2, 3]}
}
},
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
"gpu_usage": "disabled",
},
format="json",
@@ -1621,6 +1640,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"boolean_list": {"title": "It's a list of booleans", "type": "list", "required": False, "subtype": "bool", "default": [True, False, False]}
}
},
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
"gpu_usage": "disabled",
},
format="json",
@@ -1656,6 +1676,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"other_model": {"title": "Model the second", "type": "model", "default": str(self.model.id)}
}
},
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01",
"gpu_usage": "disabled",
},
format="json",
@@ -2062,7 +2083,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"worker": {
"id": str(self.worker_reco.id),
"name": "Recognizer",
@@ -2085,7 +2108,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"worker": {
"id": str(self.worker_reco.id),
"name": "Recognizer",
@@ -2147,7 +2172,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
def test_update_worker_version(self):
"""
- Update worker version artifact, configuration and state
+ Update worker version artifact, configuration, state and tag
"""
self.version_1.state = WorkerVersionState.Created
self.version_1.save()
@@ -2158,6 +2183,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"configuration": {"test": "test2"},
"docker_image_iid": "eva:unit-01",
"state": "error",
+ "tag": "eva-01"
},
format="json",
HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
@@ -2166,10 +2192,44 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
data = response.json()
self.assertEqual(data["id"], str(self.version_1.id))
self.version_1.refresh_from_db()
- self.assertEqual(data["configuration"], {"test": "test2"})
- self.assertEqual(data["docker_image_iid"], "eva:unit-01")
- self.assertEqual(data["state"], "error")
- self.assertEqual(data["gpu_usage"], "disabled")
+ self.assertEqual(self.version_1.configuration, {"test": "test2"})
+ self.assertEqual(self.version_1.docker_image_iid, "eva:unit-01")
+ self.assertEqual(self.version_1.state, WorkerVersionState.Error)
+ self.assertEqual(self.version_1.gpu_usage, FeatureUsage.Disabled)
+ self.assertEqual(self.version_1.tag, "eva-01")
+
+ def test_update_worker_version_unique_tag(self):
+ self.worker_reco.versions.create(
+ docker_image_iid="registry.fake.com/workers/worker:12345",
+ state=WorkerVersionState.Available,
+ version=12,
+ tag="eva-01"
+ )
+
+ with self.assertNumQueries(3):
+ response = self.client.patch(
+ reverse("api:version-retrieve", kwargs={"pk": str(self.version_1.id)}),
+ data={
+ "tag": "eva-01"
+ },
+ format="json",
+ HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ )
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertDictEqual(response.json(), {"tag": ["A version already exists for this worker with this tag."]})
+
+ def test_update_worker_version_empty_tag(self):
+ with self.assertNumQueries(2):
+ response = self.client.patch(
+ reverse("api:version-retrieve", kwargs={"pk": str(self.version_1.id)}),
+ data={
+ "tag": ""
+ },
+ format="json",
+ HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ )
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertDictEqual(response.json(), {"tag": ["This field may not be blank."]})
def test_cannot_update_worker_version_revision_ignored(self):
rev = self.repo.revisions.create(hash="001", message="something", author="someone")
@@ -2192,36 +2252,24 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
data = response.json()
self.assertEqual(data["id"], str(self.version_1.id))
self.version_1.refresh_from_db()
- self.assertEqual(data["revision"]["id"], str(rev.id))
+ self.assertEqual(self.version_1.revision.id, rev.id)
- def test_update_version_valid(self):
- self.version_1.state = WorkerVersionState.Created
- self.version_1.save()
-
- process = self.corpus.processes.create(
- creator=self.user,
- mode=ProcessMode.Workers,
- farm=self.farm,
- )
- process.run()
-
- response = self.client.patch(
- reverse("api:version-retrieve", kwargs={"pk": str(self.version_1.id)}),
- data={
- "configuration": {"test": "test2"},
- "docker_image_iid": "registry.nerv.co.jp/cruel/angel/thesis:latest",
- "state": "available",
- },
- format="json",
- HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
+ def test_cannot_update_worker_version_revision_url_ignored(self):
+ with self.assertNumQueries(4):
+ response = self.client.patch(
+ reverse("api:version-retrieve", kwargs={"pk": str(self.version_1.id)}),
+ data={
+ "revision_url": "https://gitlab.com/NERV/eva/commit/eva-01"
+ },
+ format="json",
+ HTTP_AUTHORIZATION=f"Ponos {self.task.token}",
+ )
+ # revision_url just gets ignored
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json()
self.assertEqual(data["id"], str(self.version_1.id))
- self.assertEqual(data["configuration"], {"test": "test2"})
- self.assertEqual(data["docker_image_iid"], "registry.nerv.co.jp/cruel/angel/thesis:latest")
- self.assertEqual(data["state"], "available")
- self.assertEqual(data["gpu_usage"], "disabled")
+ self.version_1.refresh_from_db()
+ self.assertEqual(self.version_1.revision_url, None)
def test_update_version_available_requires_docker_image(self):
self.version_1.state = WorkerVersionState.Created
@@ -2286,7 +2334,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_2.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_2.docker_image_iid,
@@ -2307,7 +2357,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_1.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_1.docker_image_iid,
@@ -2360,7 +2412,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_2.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_2.docker_image_iid,
@@ -2381,7 +2435,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_1.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_2.docker_image_iid,
@@ -2430,7 +2486,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_2.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_2.docker_image_iid,
@@ -2466,7 +2524,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_1.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_1.docker_image_iid,
@@ -2521,7 +2581,9 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
"id": str(self.version_1.id),
"configuration": {"test": 42},
"revision": None,
+ "revision_url": None,
"version": 1,
+ "tag": None,
"gpu_usage": "disabled",
"model_usage": FeatureUsage.Disabled.value,
"docker_image_iid": self.version_1.docker_image_iid,
diff --git a/arkindex/sql_validation/indexer_prefetch.sql b/arkindex/sql_validation/indexer_prefetch.sql
index 6121dbbecc6ec5cef02ed9e17a078e6be4b0eb08..8a5be55e93357c575a67a61f46c96abd5991d32e 100644
--- a/arkindex/sql_validation/indexer_prefetch.sql
+++ b/arkindex/sql_validation/indexer_prefetch.sql
@@ -69,6 +69,8 @@ SELECT "process_workerversion"."id",
"process_workerversion"."gpu_usage",
"process_workerversion"."model_usage",
"process_workerversion"."docker_image_iid",
+ "process_workerversion"."revision_url",
+ "process_workerversion"."tag",
"process_workerversion"."created",
"process_workerversion"."updated"
FROM "process_workerversion"
@@ -81,7 +83,8 @@ SELECT "process_worker"."id",
"process_worker"."description",
"process_worker"."repository_id",
"process_worker"."public",
- "process_worker"."archived"
+ "process_worker"."archived",
+ "process_worker"."repository_url"
FROM "process_worker"
WHERE "process_worker"."id" IN ('{worker_id}'::uuid);
@@ -158,6 +161,8 @@ SELECT "process_workerversion"."id",
"process_workerversion"."gpu_usage",
"process_workerversion"."model_usage",
"process_workerversion"."docker_image_iid",
+ "process_workerversion"."revision_url",
+ "process_workerversion"."tag",
"process_workerversion"."created",
"process_workerversion"."updated"
FROM "process_workerversion"
@@ -170,7 +175,8 @@ SELECT "process_worker"."id",
"process_worker"."description",
"process_worker"."repository_id",
"process_worker"."public",
- "process_worker"."archived"
+ "process_worker"."archived",
+ "process_worker"."repository_url"
FROM "process_worker"
WHERE "process_worker"."id" IN ('{worker_id}'::uuid);