diff --git a/arkindex/process/serializers/workers.py b/arkindex/process/serializers/workers.py
index 6a00e11b79a3ece24b90f75a001c0aab4b09e0fc..7581e94fecd6a876fd47529cec991d02f9fe69d1 100644
--- a/arkindex/process/serializers/workers.py
+++ b/arkindex/process/serializers/workers.py
@@ -591,10 +591,24 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
worker_slug = serializers.CharField(
max_length=100,
help_text=dedent("""
- The slug/name of the worker to which a new version will be published.
+ The slug of the worker to which a new version will be published.
If such a worker does not exist, it will be created.
"""),
)
+ worker_name = serializers.CharField(
+ max_length=100,
+ help_text="The name of the worker to which a new version will be published.",
+ )
+ worker_type = serializers.CharField(
+ max_length=100,
+ help_text="The slug of the worker type of the worker to which a new version will be published.",
+ )
+ worker_description = serializers.CharField(
+ required=False,
+ allow_null=True,
+ allow_blank=True,
+ style={"base_template": "textarea.html"},
+ )
revision_hash = serializers.CharField(max_length=50)
revision_message = serializers.CharField(required=False, default="created from docker image")
revision_author = serializers.CharField(max_length=50, required=False, default="default")
@@ -617,6 +631,9 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
# Related fields
"repository_url",
"worker_slug",
+ "worker_name",
+ "worker_type",
+ "worker_description",
"revision_hash",
"revision_message",
"revision_author",
@@ -705,23 +722,33 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
unique_fields=["repository", "name"],
)
- # Use a specific worker type in case a worker must be created
- worker_type, _ = WorkerType.objects.using("default").get_or_create(slug="docker", defaults={"display_name": "Docker"})
+ # Retrieve or create the worker type
+ worker_type, _ = WorkerType.objects.using("default").get_or_create(slug=validated_data["worker_type"], defaults={"display_name": validated_data["worker_type"]})
# Retrieve or create the worker
- worker, _ = repository.workers.using("default").get_or_create(
+ worker, created = repository.workers.using("default").get_or_create(
slug=validated_data["worker_slug"],
repository=repository,
defaults={
- "name": validated_data["worker_slug"],
- "type": worker_type,
+ "name": validated_data["worker_name"],
+ "type_id": worker_type.id,
+ "description": validated_data.get("worker_description", "")
},
)
if worker.archived:
raise ValidationError({"worker_slug": ["This worker is archived."]})
+ # Update the worker if required
+ if not created:
+ description = validated_data.get("worker_description")
+ if worker.name != validated_data["worker_name"] or worker.type_id != worker_type.id or (description is not None and worker.description != description):
+ worker.name = validated_data["worker_name"]
+ worker.type_id = worker_type.id
+ if description is not None:
+ worker.description = description
+ worker.save()
# Finally, create the worker version and mark it as available
- # If we are about the return an existing worker version, fetch the required data for the response
+ # If we are about to return an existing worker version, fetch the required data for the response
version, created = (
WorkerVersion
.objects
diff --git a/arkindex/process/tests/test_docker_worker_version.py b/arkindex/process/tests/test_docker_worker_version.py
index bea809bbacc6cf5ad02407964f8e1c85857b9133..8117a9145fc278b4b335af022c23a79b2c3d5eeb 100644
--- a/arkindex/process/tests/test_docker_worker_version.py
+++ b/arkindex/process/tests/test_docker_worker_version.py
@@ -20,8 +20,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
super().setUpTestData()
cls.repo = Repository.objects.get(url="http://my_repo.fake/workers/worker")
cls.rev = cls.repo.revisions.get()
- cls.worker = Worker.objects.get(slug="reco")
- cls.version = cls.worker.versions.get()
+ cls.worker = Worker.objects.filter(slug="reco").select_related("type").get()
+ cls.version = cls.worker.versions.select_related("revision").get()
cls.user.user_scopes.create(scope=Scope.CreateDockerWorkerVersion)
cls.worker_type = WorkerType.objects.create(slug="docker", display_name="Docker")
@@ -86,6 +86,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
},
format="json",
)
@@ -103,6 +105,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": ["This field is required."],
"revision_hash": ["This field is required."],
"worker_slug": ["This field is required."],
+ "worker_name": ["This field is required."],
+ "worker_type": ["This field is required."],
},
)
@@ -117,6 +121,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": [],
"revision_hash": "",
"worker_slug": [],
+ "worker_name": [],
+ "worker_type": [],
+ "worker_description": [],
"revision_message": [],
"revision_author": [],
"revision_references": [{"a": 133}],
@@ -142,6 +149,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"type": ["This field is required."]
}],
"worker_slug": ["Not a valid string."],
+ "worker_name": ["Not a valid string."],
+ "worker_type": ["Not a valid string."],
+ "worker_description": ["Not a valid string."],
})
def test_create_duplicated(self):
@@ -149,7 +159,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
No worker version can be created with an existing revision hash and worker slug
"""
self.client.force_login(self.user)
- with self.assertNumQueries(14):
+ with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
data={
@@ -157,7 +167,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"docker_image_iid": "some_docker_image",
"repository_url": self.repo.url,
"revision_hash": self.version.revision.hash,
- "worker_slug": self.version.worker.slug,
+ "worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
},
format="json",
)
@@ -180,6 +192,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
},
format="json",
)
@@ -194,6 +208,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
A new version can be published with only the required fields
"""
self.client.force_login(self.user)
+
with self.assertNumQueries(18):
response = self.client.post(
reverse("api:version-from-docker"),
@@ -203,6 +218,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
},
format="json",
)
@@ -243,6 +260,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
A new version can be published with the optional values
"""
self.client.force_login(self.user)
+
with self.assertNumQueries(19):
response = self.client.post(
reverse("api:version-from-docker"),
@@ -252,6 +270,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
@@ -304,6 +324,79 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
# Existing repository memberships are not updated
self.assertFalse(self.repo.memberships.exists())
+ def test_create_update_worker(self):
+ """
+ Creating a new worker version can update the worker, and create a new worker type
+ """
+ self.client.force_login(self.user)
+
+ with self.assertNumQueries(23):
+ response = self.client.post(
+ reverse("api:version-from-docker"),
+ data={
+ "configuration": {"test": "A"},
+ "docker_image_iid": "e" * 512,
+ "repository_url": self.repo.url,
+ "revision_hash": "new_revision_hash",
+ "worker_slug": self.worker.slug,
+ "worker_name": "A New Name",
+ "worker_type": "new_worker_type",
+ "worker_description": "C'est un petit val qui mousse de rayons.",
+ "revision_message": "Bruce was very clever",
+ "revision_author": "Iwan Roberts",
+ "revision_references": [
+ {"type": "branch", "name": "master"},
+ {"type": "tag", "name": "2.0"},
+ ],
+ "gpu_usage": FeatureUsage.Required.value,
+ "model_usage": FeatureUsage.Supported.value,
+ },
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+ new_worker_type = WorkerType.objects.get(slug="new_worker_type")
+ new_revision = self.repo.revisions.get(hash="new_revision_hash")
+ self.worker.refresh_from_db()
+ self.assertEqual(self.worker.description, "C'est un petit val qui mousse de rayons.")
+ refs = list(new_revision.refs.all())
+ self.assertDictEqual(
+ {ref.type: ref.name for ref in refs},
+ {GitRefType.Branch: "master", GitRefType.Tag: "2.0"},
+ )
+ new_version = new_revision.versions.get()
+ self.assertDictEqual(response.json(), {
+ "id": str(new_version.id),
+ "configuration": {"test": "A"},
+ "docker_image": None,
+ "docker_image_iid": "e" * 512,
+ "docker_image_name": new_version.docker_image_name,
+ "gpu_usage": FeatureUsage.Required.value,
+ "model_usage": FeatureUsage.Supported.value,
+ "revision": {
+ "id": str(new_revision.id),
+ "author": "Iwan Roberts",
+ "commit_url": "http://my_repo.fake/workers/worker/commit/new_revision_hash",
+ "created": new_revision.created.isoformat().replace("+00:00", "Z"),
+ "hash": "new_revision_hash",
+ "message": "Bruce was very clever",
+ "refs": [
+ {"id": str(ref.id), "name": ref.name, "type": ref.type.value}
+ for ref in refs
+ ],
+ },
+ "version": None,
+ "created": new_version.created.isoformat().replace("+00:00", "Z"),
+ "state": "available",
+ "worker": {
+ "id": str(self.worker.id),
+ "name": "A New Name",
+ "slug": "reco",
+ "type": new_worker_type.slug
+ }
+ })
+ # Existing repository memberships are not updated
+ self.assertFalse(self.repo.memberships.exists())
+
def test_create_update_git_ref(self):
"""
Existing GitRefs on the repository on a different revision
@@ -331,6 +424,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
@@ -421,7 +516,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"docker_image_iid": "some_docker_image",
"repository_url": self.repo.url,
"revision_hash": self.version.revision.hash,
- "worker_slug": self.version.worker.slug,
+ "worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
"revision_references": [
{"type": "tag", "name": "9.9.9.9.9.9"},
],
@@ -474,11 +571,11 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
def test_create_git_stack(self):
"""
All git references can be created including repository, revision, gitrefs and worker
- including the default worker type
+ and worker type
"""
self.worker_type.delete()
self.client.force_login(self.user)
- with self.assertNumQueries(29):
+ with self.assertNumQueries(30):
response = self.client.post(
reverse("api:version-from-docker"),
data={
@@ -487,6 +584,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": "https://gitlab.test.arkindex.org/project/",
"revision_hash": "deadbeef",
"worker_slug": "new_gen_classifier",
+ "worker_name": "Classifier New Generation",
+ "worker_type": "a_new_type",
"revision_references": [
{"type": "branch", "name": "master"},
],
@@ -521,7 +620,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"worker": {
"id": str(new_version.worker.id),
- "name": "new_gen_classifier",
+ "name": "Classifier New Generation",
"slug": "new_gen_classifier",
"type": str(new_worker_type),
}
@@ -533,7 +632,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.assertEqual(new_revision.repo, new_repo)
self.assertEqual(new_ref.name, "master")
self.assertEqual(new_ref.repository, new_repo)
- self.assertEqual(new_version.worker.name, "new_gen_classifier")
+ self.assertEqual(new_version.worker.name, "Classifier New Generation")
self.assertEqual(new_version.worker.slug, "new_gen_classifier")
self.assertEqual(new_version.worker.public, False)
self.assertEqual(new_version.worker.repository, new_repo)
@@ -542,8 +641,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.assertEqual(new_version.docker_image_iid, "docker_image_42")
self.assertEqual(new_version.gpu_usage, FeatureUsage.Disabled)
self.assertEqual(new_version.model_usage, FeatureUsage.Disabled)
- self.assertEqual(new_worker_type.slug, "docker")
- self.assertEqual(new_worker_type.display_name, "Docker")
+ self.assertEqual(new_worker_type.slug, "a_new_type")
+ self.assertEqual(new_worker_type.display_name, "a_new_type")
# User is granted an admin role on the repository
self.assertListEqual(list(new_repo.memberships.values_list("user", "level")), [
(self.user.id, Role.Admin.value)
@@ -564,6 +663,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": {
"demo_integer": {"title": "Demo Integer", "type": "int", "required": True, "default": 1},
@@ -707,6 +808,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": user_configuration
},
@@ -744,6 +847,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
+ "worker_name": self.worker.name,
+ "worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": {
"param": {"title": "param", **params}