Skip to content
Snippets Groups Projects
Commit 78d3ced9 authored by ml bonhomme's avatar ml bonhomme :bee:
Browse files

Support all worker attributes on CreateDockerWorkerVersion

parent 54a2c429
No related branches found
No related tags found
No related merge requests found
This commit is part of merge request !2299. Comments created here will be created in the context of that merge request.
Hide whitespace changes
Inline Side-by-side
arkindex/process/serializers/workers.py
+ 28
− 7
View file @ 78d3ced9
......@@ -591,10 +591,19 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
worker_slug = serializers.CharField(
max_length=100,
help_text=dedent("""
The slug/name of the worker to which a new version will be published.
The slug of the worker to which a new version will be published.
If such a worker does not exist, it will be created.
"""),
)
worker_name = serializers.CharField(
max_length=100,
help_text="The name of the worker to which a new version will be published.",
)
worker_type = serializers.CharField(
max_length=100,
help_text="The slug of the worker type of the worker to which a new version will be published.",
)
worker_description = serializers.CharField(required=False, style={"base_template": "textarea.html"})
revision_hash = serializers.CharField(max_length=50)
revision_message = serializers.CharField(required=False, default="created from docker image")
revision_author = serializers.CharField(max_length=50, required=False, default="default")
......@@ -617,6 +626,9 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
# Related fields
"repository_url",
"worker_slug",
"worker_name",
"worker_type",
"worker_description",
"revision_hash",
"revision_message",
"revision_author",
......@@ -705,23 +717,32 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
unique_fields=["repository", "name"],
)
# Use a specific worker type in case a worker must be created
worker_type, _ = WorkerType.objects.using("default").get_or_create(slug="docker", defaults={"display_name": "Docker"})
# Retrieve or create the worker type
worker_type, _ = WorkerType.objects.using("default").get_or_create(slug=validated_data["worker_type"], defaults={"display_name": validated_data["worker_type"]})
# Retrieve or create the worker
worker, _ = repository.workers.using("default").get_or_create(
worker, created = repository.workers.using("default").get_or_create(
slug=validated_data["worker_slug"],
repository=repository,
defaults={
"name": validated_data["worker_slug"],
"type": worker_type,
"name": validated_data["worker_name"],
"type_id": worker_type.id,
"description": validated_data.get("worker_description", "")
},
)
if worker.archived:
raise ValidationError({"worker_slug": ["This worker is archived."]})
# Update the worker if required
if not created:
description = validated_data.get("worker_description")
if worker.name != validated_data["worker_name"] or worker.type_id != worker_type.id or (description and worker.description != description):
worker.name = validated_data["worker_name"]
worker.type_id = worker_type.id
worker.description = description if description else worker.description
worker.save()
# Finally, create the worker version and mark it as available
# If we are about the return an existing worker version, fetch the required data for the response
# If we are about to return an existing worker version, fetch the required data for the response
version, created = (
WorkerVersion
.objects
......
arkindex/process/tests/test_docker_worker_version.py
+ 134
− 11
View file @ 78d3ced9
......@@ -78,6 +78,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"""
self.assertEqual(self.superuser.user_scopes.count(), 0)
self.client.force_login(self.superuser)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
response = self.client.post(
reverse("api:version-from-docker"),
data={
......@@ -86,6 +88,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -103,6 +107,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": ["This field is required."],
"revision_hash": ["This field is required."],
"worker_slug": ["This field is required."],
"worker_name": ["This field is required."],
"worker_type": ["This field is required."],
},
)
......@@ -117,6 +123,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": [],
"revision_hash": "",
"worker_slug": [],
"worker_name": [],
"worker_type": [],
"worker_description": [],
"revision_message": [],
"revision_author": [],
"revision_references": [{"a": 133}],
......@@ -142,22 +151,31 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"type": ["This field is required."]
}],
"worker_slug": ["Not a valid string."],
"worker_name": ["Not a valid string."],
"worker_type": ["Not a valid string."],
"worker_description": ["Not a valid string."],
})
def test_create_duplicated(self):
"""
No worker version can be created with an existing revision hash and worker slug
"""
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
revision = self.version.revision
worker = self.version.worker
type_slug = worker.type.slug
self.client.force_login(self.user)
with self.assertNumQueries(14):
with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
data={
"configuration": {},
"docker_image_iid": "some_docker_image",
"repository_url": self.repo.url,
"revision_hash": self.version.revision.hash,
"worker_slug": self.version.worker.slug,
"revision_hash": revision.hash,
"worker_slug": worker.slug,
"worker_name": worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -171,6 +189,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.worker.archived = datetime.now(timezone.utc)
self.worker.save()
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -180,6 +200,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -194,6 +216,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
A new version can be published with only the required fields
"""
self.client.force_login(self.user)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(18):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -203,6 +228,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -243,6 +270,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
A new version can be published with the optional values
"""
self.client.force_login(self.user)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(19):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -252,6 +282,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
......@@ -304,6 +336,79 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
# Existing repository memberships are not updated
self.assertFalse(self.repo.memberships.exists())
def test_create_update_worker(self):
"""
Creating a new worker version can update the worker, and create a new worker type
"""
self.client.force_login(self.user)
with self.assertNumQueries(23):
response = self.client.post(
reverse("api:version-from-docker"),
data={
"configuration": {"test": "A"},
"docker_image_iid": "e" * 512,
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": "A New Name",
"worker_type": "new_worker_type",
"worker_description": "C'est un petit val qui mousse de rayons.",
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
{"type": "branch", "name": "master"},
{"type": "tag", "name": "2.0"},
],
"gpu_usage": FeatureUsage.Required.value,
"model_usage": FeatureUsage.Supported.value,
},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
new_worker_type = WorkerType.objects.get(slug="new_worker_type")
new_revision = self.repo.revisions.get(hash="new_revision_hash")
self.worker.refresh_from_db()
self.assertEqual(self.worker.description, "C'est un petit val qui mousse de rayons.")
refs = list(new_revision.refs.all())
self.assertDictEqual(
{ref.type: ref.name for ref in refs},
{GitRefType.Branch: "master", GitRefType.Tag: "2.0"},
)
new_version = new_revision.versions.get()
self.assertDictEqual(response.json(), {
"id": str(new_version.id),
"configuration": {"test": "A"},
"docker_image": None,
"docker_image_iid": "e" * 512,
"docker_image_name": new_version.docker_image_name,
"gpu_usage": FeatureUsage.Required.value,
"model_usage": FeatureUsage.Supported.value,
"revision": {
"id": str(new_revision.id),
"author": "Iwan Roberts",
"commit_url": "http://my_repo.fake/workers/worker/commit/new_revision_hash",
"created": new_revision.created.isoformat().replace("+00:00", "Z"),
"hash": "new_revision_hash",
"message": "Bruce was very clever",
"refs": [
{"id": str(ref.id), "name": ref.name, "type": ref.type.value}
for ref in refs
],
},
"version": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
"id": str(self.worker.id),
"name": "A New Name",
"slug": "reco",
"type": new_worker_type.slug
}
})
# Existing repository memberships are not updated
self.assertFalse(self.repo.memberships.exists())
def test_create_update_git_ref(self):
"""
Existing GitRefs on the repository on a different revision
......@@ -322,6 +427,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
)
self.client.force_login(self.user)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(19):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -331,6 +438,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
......@@ -413,6 +522,10 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
repository=self.version.revision.repo,
)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
revision = self.version.revision
worker = self.version.worker
type_slug = worker.type.slug
with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -420,8 +533,10 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"configuration": {},
"docker_image_iid": "some_docker_image",
"repository_url": self.repo.url,
"revision_hash": self.version.revision.hash,
"worker_slug": self.version.worker.slug,
"revision_hash": revision.hash,
"worker_slug": worker.slug,
"worker_name": worker.name,
"worker_type": type_slug,
"revision_references": [
{"type": "tag", "name": "9.9.9.9.9.9"},
],
......@@ -474,11 +589,11 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
def test_create_git_stack(self):
"""
All git references can be created including repository, revision, gitrefs and worker
including the default worker type
and worker type
"""
self.worker_type.delete()
self.client.force_login(self.user)
with self.assertNumQueries(29):
with self.assertNumQueries(30):
response = self.client.post(
reverse("api:version-from-docker"),
data={
......@@ -487,6 +602,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": "https://gitlab.test.arkindex.org/project/",
"revision_hash": "deadbeef",
"worker_slug": "new_gen_classifier",
"worker_name": "Classifier New Generation",
"worker_type": "a_new_type",
"revision_references": [
{"type": "branch", "name": "master"},
],
......@@ -521,7 +638,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"worker": {
"id": str(new_version.worker.id),
"name": "new_gen_classifier",
"name": "Classifier New Generation",
"slug": "new_gen_classifier",
"type": str(new_worker_type),
}
......@@ -533,7 +650,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.assertEqual(new_revision.repo, new_repo)
self.assertEqual(new_ref.name, "master")
self.assertEqual(new_ref.repository, new_repo)
self.assertEqual(new_version.worker.name, "new_gen_classifier")
self.assertEqual(new_version.worker.name, "Classifier New Generation")
self.assertEqual(new_version.worker.slug, "new_gen_classifier")
self.assertEqual(new_version.worker.public, False)
self.assertEqual(new_version.worker.repository, new_repo)
......@@ -542,8 +659,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.assertEqual(new_version.docker_image_iid, "docker_image_42")
self.assertEqual(new_version.gpu_usage, FeatureUsage.Disabled)
self.assertEqual(new_version.model_usage, FeatureUsage.Disabled)
self.assertEqual(new_worker_type.slug, "docker")
self.assertEqual(new_worker_type.display_name, "Docker")
self.assertEqual(new_worker_type.slug, "a_new_type")
self.assertEqual(new_worker_type.display_name, "a_new_type")
# User is granted an admin role on the repository
self.assertListEqual(list(new_repo.memberships.values_list("user", "level")), [
(self.user.id, Role.Admin.value)
......@@ -564,6 +681,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": {
"demo_integer": {"title": "Demo Integer", "type": "int", "required": True, "default": 1},
......@@ -707,6 +826,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": user_configuration
},
......@@ -744,6 +865,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": {
"param": {"title": "param", **params}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment