Skip to content
Snippets Groups Projects
Commit 78d3ced9 authored by ml bonhomme's avatar ml bonhomme :bee:
Browse files

Support all worker attributes on CreateDockerWorkerVersion

parent 54a2c429
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
arkindex/process/serializers/workers.py
+ 28
− 7
View file @ 78d3ced9
......@@ -591,10 +591,19 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
worker_slug = serializers.CharField(
max_length=100,
help_text=dedent("""
The slug/name of the worker to which a new version will be published.
The slug of the worker to which a new version will be published.
If such a worker does not exist, it will be created.
"""),
)
worker_name = serializers.CharField(
max_length=100,
help_text="The name of the worker to which a new version will be published.",
)
worker_type = serializers.CharField(
max_length=100,
help_text="The slug of the worker type of the worker to which a new version will be published.",
)
worker_description = serializers.CharField(required=False, style={"base_template": "textarea.html"})
revision_hash = serializers.CharField(max_length=50)
revision_message = serializers.CharField(required=False, default="created from docker image")
revision_author = serializers.CharField(max_length=50, required=False, default="default")
......@@ -617,6 +626,9 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
# Related fields
"repository_url",
"worker_slug",
"worker_name",
"worker_type",
"worker_description",
"revision_hash",
"revision_message",
"revision_author",
......@@ -705,23 +717,32 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
unique_fields=["repository", "name"],
)
# Use a specific worker type in case a worker must be created
worker_type, _ = WorkerType.objects.using("default").get_or_create(slug="docker", defaults={"display_name": "Docker"})
# Retrieve or create the worker type
worker_type, _ = WorkerType.objects.using("default").get_or_create(slug=validated_data["worker_type"], defaults={"display_name": validated_data["worker_type"]})
# Retrieve or create the worker
worker, _ = repository.workers.using("default").get_or_create(
worker, created = repository.workers.using("default").get_or_create(
slug=validated_data["worker_slug"],
repository=repository,
defaults={
"name": validated_data["worker_slug"],
"type": worker_type,
"name": validated_data["worker_name"],
"type_id": worker_type.id,
"description": validated_data.get("worker_description", "")
},
)
if worker.archived:
raise ValidationError({"worker_slug": ["This worker is archived."]})
# Update the worker if required
if not created:
description = validated_data.get("worker_description")
if worker.name != validated_data["worker_name"] or worker.type_id != worker_type.id or (description and worker.description != description):
worker.name = validated_data["worker_name"]
worker.type_id = worker_type.id
worker.description = description if description else worker.description
worker.save()
# Finally, create the worker version and mark it as available
# If we are about the return an existing worker version, fetch the required data for the response
# If we are about to return an existing worker version, fetch the required data for the response
version, created = (
WorkerVersion
.objects
......
arkindex/process/tests/test_docker_worker_version.py
+ 134
− 11
View file @ 78d3ced9
......@@ -78,6 +78,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"""
self.assertEqual(self.superuser.user_scopes.count(), 0)
self.client.force_login(self.superuser)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
response = self.client.post(
reverse("api:version-from-docker"),
data={
......@@ -86,6 +88,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -103,6 +107,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": ["This field is required."],
"revision_hash": ["This field is required."],
"worker_slug": ["This field is required."],
"worker_name": ["This field is required."],
"worker_type": ["This field is required."],
},
)
......@@ -117,6 +123,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": [],
"revision_hash": "",
"worker_slug": [],
"worker_name": [],
"worker_type": [],
"worker_description": [],
"revision_message": [],
"revision_author": [],
"revision_references": [{"a": 133}],
......@@ -142,22 +151,31 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"type": ["This field is required."]
}],
"worker_slug": ["Not a valid string."],
"worker_name": ["Not a valid string."],
"worker_type": ["Not a valid string."],
"worker_description": ["Not a valid string."],
})
def test_create_duplicated(self):
"""
No worker version can be created with an existing revision hash and worker slug
"""
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
revision = self.version.revision
worker = self.version.worker
type_slug = worker.type.slug
self.client.force_login(self.user)
with self.assertNumQueries(14):
with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
data={
"configuration": {},
"docker_image_iid": "some_docker_image",
"repository_url": self.repo.url,
"revision_hash": self.version.revision.hash,
"worker_slug": self.version.worker.slug,
"revision_hash": revision.hash,
"worker_slug": worker.slug,
"worker_name": worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -171,6 +189,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.worker.archived = datetime.now(timezone.utc)
self.worker.save()
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -180,6 +200,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -194,6 +216,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
A new version can be published with only the required fields
"""
self.client.force_login(self.user)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(18):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -203,6 +228,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
},
format="json",
)
......@@ -243,6 +270,9 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
A new version can be published with the optional values
"""
self.client.force_login(self.user)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(19):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -252,6 +282,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
......@@ -304,6 +336,79 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
# Existing repository memberships are not updated
self.assertFalse(self.repo.memberships.exists())
def test_create_update_worker(self):
"""
Creating a new worker version can update the worker, and create a new worker type
"""
self.client.force_login(self.user)
with self.assertNumQueries(23):
response = self.client.post(
reverse("api:version-from-docker"),
data={
"configuration": {"test": "A"},
"docker_image_iid": "e" * 512,
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": "A New Name",
"worker_type": "new_worker_type",
"worker_description": "C'est un petit val qui mousse de rayons.",
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
{"type": "branch", "name": "master"},
{"type": "tag", "name": "2.0"},
],
"gpu_usage": FeatureUsage.Required.value,
"model_usage": FeatureUsage.Supported.value,
},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
new_worker_type = WorkerType.objects.get(slug="new_worker_type")
new_revision = self.repo.revisions.get(hash="new_revision_hash")
self.worker.refresh_from_db()
self.assertEqual(self.worker.description, "C'est un petit val qui mousse de rayons.")
refs = list(new_revision.refs.all())
self.assertDictEqual(
{ref.type: ref.name for ref in refs},
{GitRefType.Branch: "master", GitRefType.Tag: "2.0"},
)
new_version = new_revision.versions.get()
self.assertDictEqual(response.json(), {
"id": str(new_version.id),
"configuration": {"test": "A"},
"docker_image": None,
"docker_image_iid": "e" * 512,
"docker_image_name": new_version.docker_image_name,
"gpu_usage": FeatureUsage.Required.value,
"model_usage": FeatureUsage.Supported.value,
"revision": {
"id": str(new_revision.id),
"author": "Iwan Roberts",
"commit_url": "http://my_repo.fake/workers/worker/commit/new_revision_hash",
"created": new_revision.created.isoformat().replace("+00:00", "Z"),
"hash": "new_revision_hash",
"message": "Bruce was very clever",
"refs": [
{"id": str(ref.id), "name": ref.name, "type": ref.type.value}
for ref in refs
],
},
"version": None,
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"state": "available",
"worker": {
"id": str(self.worker.id),
"name": "A New Name",
"slug": "reco",
"type": new_worker_type.slug
}
})
# Existing repository memberships are not updated
self.assertFalse(self.repo.memberships.exists())
def test_create_update_git_ref(self):
"""
Existing GitRefs on the repository on a different revision
......@@ -322,6 +427,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
)
self.client.force_login(self.user)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
type_slug = self.version.worker.type.slug
with self.assertNumQueries(19):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -331,6 +438,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": type_slug,
"revision_message": "Bruce was very clever",
"revision_author": "Iwan Roberts",
"revision_references": [
......@@ -413,6 +522,10 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
repository=self.version.revision.repo,
)
# Keep this out of the self.assertNumQueries so as not to add irrelevant queries
revision = self.version.revision
worker = self.version.worker
type_slug = worker.type.slug
with self.assertNumQueries(13):
response = self.client.post(
reverse("api:version-from-docker"),
......@@ -420,8 +533,10 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"configuration": {},
"docker_image_iid": "some_docker_image",
"repository_url": self.repo.url,
"revision_hash": self.version.revision.hash,
"worker_slug": self.version.worker.slug,
"revision_hash": revision.hash,
"worker_slug": worker.slug,
"worker_name": worker.name,
"worker_type": type_slug,
"revision_references": [
{"type": "tag", "name": "9.9.9.9.9.9"},
],
......@@ -474,11 +589,11 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
def test_create_git_stack(self):
"""
All git references can be created including repository, revision, gitrefs and worker
including the default worker type
and worker type
"""
self.worker_type.delete()
self.client.force_login(self.user)
with self.assertNumQueries(29):
with self.assertNumQueries(30):
response = self.client.post(
reverse("api:version-from-docker"),
data={
......@@ -487,6 +602,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": "https://gitlab.test.arkindex.org/project/",
"revision_hash": "deadbeef",
"worker_slug": "new_gen_classifier",
"worker_name": "Classifier New Generation",
"worker_type": "a_new_type",
"revision_references": [
{"type": "branch", "name": "master"},
],
......@@ -521,7 +638,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"created": new_version.created.isoformat().replace("+00:00", "Z"),
"worker": {
"id": str(new_version.worker.id),
"name": "new_gen_classifier",
"name": "Classifier New Generation",
"slug": "new_gen_classifier",
"type": str(new_worker_type),
}
......@@ -533,7 +650,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.assertEqual(new_revision.repo, new_repo)
self.assertEqual(new_ref.name, "master")
self.assertEqual(new_ref.repository, new_repo)
self.assertEqual(new_version.worker.name, "new_gen_classifier")
self.assertEqual(new_version.worker.name, "Classifier New Generation")
self.assertEqual(new_version.worker.slug, "new_gen_classifier")
self.assertEqual(new_version.worker.public, False)
self.assertEqual(new_version.worker.repository, new_repo)
......@@ -542,8 +659,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
self.assertEqual(new_version.docker_image_iid, "docker_image_42")
self.assertEqual(new_version.gpu_usage, FeatureUsage.Disabled)
self.assertEqual(new_version.model_usage, FeatureUsage.Disabled)
self.assertEqual(new_worker_type.slug, "docker")
self.assertEqual(new_worker_type.display_name, "Docker")
self.assertEqual(new_worker_type.slug, "a_new_type")
self.assertEqual(new_worker_type.display_name, "a_new_type")
# User is granted an admin role on the repository
self.assertListEqual(list(new_repo.memberships.values_list("user", "level")), [
(self.user.id, Role.Admin.value)
......@@ -564,6 +681,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": {
"demo_integer": {"title": "Demo Integer", "type": "int", "required": True, "default": 1},
......@@ -707,6 +826,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": user_configuration
},
......@@ -744,6 +865,8 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
"repository_url": self.repo.url,
"revision_hash": "new_revision_hash",
"worker_slug": self.worker.slug,
"worker_name": self.worker.name,
"worker_type": self.worker.type.slug,
"configuration": {
"user_configuration": {
"param": {"title": "param", **params}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment