diff --git a/README.md b/README.md
index 0f7632ab3b540bd95957a90598a163a16f13fba2..0915fb3dfe17f1fb33abe1e505a469ac822c8552 100644
--- a/README.md
+++ b/README.md
@@ -43,26 +43,6 @@ You will need to edit the ImageMagick policy file to get PDF and Image imports t
The line that sets the PDF policy is `<policy domain="coder" rights="none" pattern="PDF" />`. Replace `none` with `read|write` for it to work. See [this StackOverflow question](https://stackoverflow.com/questions/52998331) for more info.
-### GitLab OAuth setup
-
-Arkindex uses OAuth to let a user connect their GitLab account(s) and register Git repositories. In local development, you will need to register Arkindex as a GitLab OAuth application for it to work.
-
-Go to GitLab's [Applications settings](https://gitlab.teklia.com/profile/applications) and create a new application with the `api` scope and add the following callback URIs:
-
-```
-http://127.0.0.1:8000/api/v1/oauth/providers/gitlab/callback/
-http://ark.localhost:8000/api/v1/oauth/providers/gitlab/callback/
-https://ark.localhost/api/v1/oauth/providers/gitlab/callback/
-```
-
-Once the application is created, GitLab will provide you with an application ID and a secret. Use the `arkindex/config.yml` file to set them:
-
-```yaml
-gitlab:
- app_id: 24cacf5004bf68ae9daad19a5bba391d85ad1cb0b31366e89aec86fad0ab16cb
- app_secret: 9d96d9d5b1addd7e7e6119a23b1e5b5f68545312bfecb21d1cdc6af22b8628b8
-```
-
### Local image server
Arkindex splits up image URLs in their image server and the image path. For example, a IIIF server at `http://iiif.irht.cnrs.fr/iiif/` and an image at `/Paris/JJ042/1.jpg` would be represented as an ImageServer instance holding one Image. Since Arkindex has a local IIIF server for image uploads and thumbnails, a special instance of ImageServer is required to point to this local server. In local development, this server should be available at `https://ark.localhost/iiif`. You will therefore need to create an ImageServer via the Django admin or the Django shell with this URL. To set the local server ID, you can add a custom setting in `arkindex/config.yml`:
@@ -161,9 +141,6 @@ SHELL_PLUS_POST_IMPORTS = [
)),
('arkindex.project.aws', (
'S3FileStatus',
- )),
- ('arkindex.users.models', (
- 'OAuthStatus',
))
]
```
diff --git a/arkindex/documents/fixtures/data.json b/arkindex/documents/fixtures/data.json
index efd6ebcd8817dca400f85166fbe46631cc9d803d..b515da387b3bc5741722cb1c5c03a4014682f7b8 100644
--- a/arkindex/documents/fixtures/data.json
+++ b/arkindex/documents/fixtures/data.json
@@ -1,7 +1,7 @@
[
{
"model": "process.process",
- "pk": "7b0b9c75-bffc-42f5-a646-c81e8197d171",
+ "pk": "55e92ea6-625a-4c30-9724-b1f327941cac",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
@@ -36,19 +36,19 @@
},
{
"model": "process.process",
- "pk": "8cec0605-06e6-4b8f-ba7d-3ec9f9f2a409",
+ "pk": "6f170a31-9cf3-4fef-a684-ce8c8a4000d2",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "name": "Process fixture",
- "creator": 2,
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "mode": "workers",
+ "name": null,
+ "creator": 1,
+ "corpus": null,
+ "mode": "repository",
"revision": null,
"activity_state": "disabled",
"started": null,
"finished": null,
- "farm": "409b6859-63b9-41f2-8449-ba737bca6624",
+ "farm": "d4639943-3bc5-4839-ba11-bac03818050c",
"element": null,
"folder_type": null,
"element_type": null,
@@ -71,19 +71,19 @@
},
{
"model": "process.process",
- "pk": "9a08ea15-07be-4746-a0d9-7acca68e5c1b",
+ "pk": "6f8334b3-b89c-48ef-b69f-255a5c4f868d",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"name": null,
"creator": 1,
"corpus": null,
- "mode": "repository",
+ "mode": "local",
"revision": null,
"activity_state": "disabled",
"started": null,
"finished": null,
- "farm": "409b6859-63b9-41f2-8449-ba737bca6624",
+ "farm": null,
"element": null,
"folder_type": null,
"element_type": null,
@@ -106,19 +106,19 @@
},
{
"model": "process.process",
- "pk": "b4b4b4af-c221-4c47-a784-cc1bac1a99b1",
+ "pk": "d9f1b26c-6aae-4e22-a9cc-d218a245794e",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "name": null,
- "creator": 1,
- "corpus": null,
- "mode": "local",
+ "name": "Process fixture",
+ "creator": 2,
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "mode": "workers",
"revision": null,
"activity_state": "disabled",
"started": null,
"finished": null,
- "farm": null,
+ "farm": "d4639943-3bc5-4839-ba11-bac03818050c",
"element": null,
"folder_type": null,
"element_type": null,
@@ -141,29 +141,25 @@
},
{
"model": "process.repository",
- "pk": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
+ "pk": "c32a2335-9132-402b-9a3b-bd76b0140c1a",
"fields": {
- "url": "http://my_repo.fake/workers/worker",
- "hook_token": "worker-hook-token",
- "credentials": "217e2e72-f917-46de-9760-0e140bec08eb"
+ "url": "http://gitlab/repo"
}
},
{
"model": "process.repository",
- "pk": "e4f4a2e6-b23b-48d2-bf86-b60c2b7d2810",
+ "pk": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
"fields": {
- "url": "http://gitlab/repo",
- "hook_token": "hook-token",
- "credentials": "217e2e72-f917-46de-9760-0e140bec08eb"
+ "url": "http://my_repo.fake/workers/worker"
}
},
{
"model": "process.revision",
- "pk": "2b024a21-5655-45b7-9fe0-d27b58a6a2d2",
+ "pk": "97306172-bca6-46a6-9789-4b084e09c008",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "repo": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
+ "repo": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
"hash": "1337",
"message": "My w0rk3r",
"author": "Test user"
@@ -171,11 +167,11 @@
},
{
"model": "process.revision",
- "pk": "f7845dce-ce21-49b6-9962-3eb9cec30a6d",
+ "pk": "c5cf2885-3440-4838-be93-d8211c8a811c",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "repo": "e4f4a2e6-b23b-48d2-bf86-b60c2b7d2810",
+ "repo": "c32a2335-9132-402b-9a3b-bd76b0140c1a",
"hash": "42",
"message": "Salve",
"author": "Some user"
@@ -183,93 +179,105 @@
},
{
"model": "process.worker",
- "pk": "08ef0527-5af6-45c7-a84c-d9d69697e5ca",
+ "pk": "03ca969c-2fee-4a4c-b2f7-c283fb72bf91",
+ "fields": {
+ "name": "Worker requiring a GPU",
+ "slug": "worker-gpu",
+ "type": "7dee488e-7a48-434b-9163-6396972bff7f",
+ "description": "",
+ "repository": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
+ "public": false,
+ "archived": null
+ }
+},
+{
+ "model": "process.worker",
+ "pk": "081bfea3-f8eb-4070-9cb4-6cc4e7b260b8",
"fields": {
"name": "Recognizer",
"slug": "reco",
- "type": "f089f069-68e8-48cc-a168-19a07a8681f8",
- "repository": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
- "public": false
+ "type": "250cfaca-9733-42ef-a265-b8e8aea73075",
+ "description": "",
+ "repository": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
+ "public": false,
+ "archived": null
}
},
{
"model": "process.worker",
- "pk": "25bfe0e8-7c7c-491e-b9b4-73498c4b5eb7",
+ "pk": "3d121605-190a-4faa-8085-e7c1bffc4297",
"fields": {
"name": "Document layout analyser",
"slug": "dla",
- "type": "36873927-4925-4eca-972b-f8dfed39cbe8",
- "repository": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
- "public": false
+ "type": "40620987-763e-4e69-baa8-5bdf28f8b6c3",
+ "description": "",
+ "repository": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
+ "public": false,
+ "archived": null
}
},
{
"model": "process.worker",
- "pk": "5222a3e0-b27c-43c2-96df-905613cfd55a",
+ "pk": "6a9a5388-c047-4bbf-9c41-86fbc578fd92",
"fields": {
"name": "Custom worker",
"slug": "custom",
- "type": "5371cc7b-b6c7-407a-a935-e97d05caf1cb",
+ "type": "a9458b4e-5ac9-4caa-b2fd-516b16222604",
+ "description": "",
"repository": null,
- "public": false
+ "public": false,
+ "archived": null
}
},
{
"model": "process.worker",
- "pk": "913ff861-730c-40e1-9d47-d5d271586c35",
+ "pk": "a9950b39-1c6d-4dde-bbd5-9545871189f4",
"fields": {
"name": "File import",
"slug": "file_import",
- "type": "ba1d9a09-48d7-4ac5-8427-bbd51775ea7c",
- "repository": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
- "public": false
- }
-},
-{
- "model": "process.worker",
- "pk": "ed512f68-389e-4e4d-8f96-103ecab3122b",
- "fields": {
- "name": "Worker requiring a GPU",
- "slug": "worker-gpu",
- "type": "fd2df8fb-cbc1-4c43-9abe-1a6db6122e59",
- "repository": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
- "public": false
+ "type": "6772567a-5429-4d97-b1aa-8c779d30c68f",
+ "description": "",
+ "repository": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
+ "public": false,
+ "archived": null
}
},
{
"model": "process.worker",
- "pk": "fc99199f-0f3a-4c0d-a033-db7f151910b1",
+ "pk": "ddeda096-4d97-45c5-95fc-600397638cb7",
"fields": {
"name": "Generic worker with a Model",
"slug": "generic",
- "type": "f089f069-68e8-48cc-a168-19a07a8681f8",
- "repository": "9c6b9576-23e1-4efb-88ce-8a15550b06cb",
- "public": false
+ "type": "250cfaca-9733-42ef-a265-b8e8aea73075",
+ "description": "",
+ "repository": "eaa88eb6-31f6-464c-bba1-d64724703d2f",
+ "public": false,
+ "archived": null
}
},
{
"model": "process.workertype",
- "pk": "36873927-4925-4eca-972b-f8dfed39cbe8",
+ "pk": "250cfaca-9733-42ef-a265-b8e8aea73075",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "slug": "dla",
- "display_name": "Document Layout Analysis"
+ "slug": "recognizer",
+ "display_name": "Recognizer"
}
},
{
"model": "process.workertype",
- "pk": "5371cc7b-b6c7-407a-a935-e97d05caf1cb",
+ "pk": "40620987-763e-4e69-baa8-5bdf28f8b6c3",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "slug": "custom",
- "display_name": "Custom"
+ "slug": "dla",
+ "display_name": "Document Layout Analysis"
}
},
{
"model": "process.workertype",
- "pk": "ba1d9a09-48d7-4ac5-8427-bbd51775ea7c",
+ "pk": "6772567a-5429-4d97-b1aa-8c779d30c68f",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
@@ -279,38 +287,36 @@
},
{
"model": "process.workertype",
- "pk": "f089f069-68e8-48cc-a168-19a07a8681f8",
+ "pk": "7dee488e-7a48-434b-9163-6396972bff7f",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "slug": "recognizer",
- "display_name": "Recognizer"
+ "slug": "worker",
+ "display_name": "Worker requiring a GPU"
}
},
{
"model": "process.workertype",
- "pk": "fd2df8fb-cbc1-4c43-9abe-1a6db6122e59",
+ "pk": "a9458b4e-5ac9-4caa-b2fd-516b16222604",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "slug": "worker",
- "display_name": "Worker requiring a GPU"
+ "slug": "custom",
+ "display_name": "Custom"
}
},
{
"model": "process.workerversion",
- "pk": "06e97787-7dbd-424b-bd3e-40d4de6b107f",
+ "pk": "28cc29d7-c2c2-41d2-b051-4553f9535ea9",
"fields": {
- "worker": "ed512f68-389e-4e4d-8f96-103ecab3122b",
- "revision": "2b024a21-5655-45b7-9fe0-d27b58a6a2d2",
+ "worker": "a9950b39-1c6d-4dde-bbd5-9545871189f4",
+ "revision": "97306172-bca6-46a6-9789-4b084e09c008",
"version": null,
- "configuration": {
- "test": 42
- },
+ "configuration": {},
"state": "available",
- "gpu_usage": "required",
+ "gpu_usage": "disabled",
"model_usage": "disabled",
- "docker_image": "a93301c3-bdac-4d06-ba22-b7066decd282",
+ "docker_image": "27166636-75bd-4296-adce-ba8fa09a36c5",
"docker_image_iid": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
@@ -318,18 +324,18 @@
},
{
"model": "process.workerversion",
- "pk": "5bb8f594-dff8-4145-8634-9ba88ff8a61d",
+ "pk": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"fields": {
- "worker": "5222a3e0-b27c-43c2-96df-905613cfd55a",
- "revision": null,
- "version": 1,
+ "worker": "081bfea3-f8eb-4070-9cb4-6cc4e7b260b8",
+ "revision": "97306172-bca6-46a6-9789-4b084e09c008",
+ "version": null,
"configuration": {
- "custom": "value"
+ "test": 42
},
- "state": "created",
+ "state": "available",
"gpu_usage": "disabled",
"model_usage": "disabled",
- "docker_image": null,
+ "docker_image": "27166636-75bd-4296-adce-ba8fa09a36c5",
"docker_image_iid": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
@@ -337,16 +343,18 @@
},
{
"model": "process.workerversion",
- "pk": "6610938d-c68c-445c-a4cc-d21e574a8958",
+ "pk": "3e12e240-0b7e-43f1-8e63-0ccd0b851548",
"fields": {
- "worker": "913ff861-730c-40e1-9d47-d5d271586c35",
- "revision": "2b024a21-5655-45b7-9fe0-d27b58a6a2d2",
+ "worker": "ddeda096-4d97-45c5-95fc-600397638cb7",
+ "revision": "97306172-bca6-46a6-9789-4b084e09c008",
"version": null,
- "configuration": {},
+ "configuration": {
+ "test": 42
+ },
"state": "available",
"gpu_usage": "disabled",
- "model_usage": "disabled",
- "docker_image": "a93301c3-bdac-4d06-ba22-b7066decd282",
+ "model_usage": "required",
+ "docker_image": "27166636-75bd-4296-adce-ba8fa09a36c5",
"docker_image_iid": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
@@ -354,18 +362,18 @@
},
{
"model": "process.workerversion",
- "pk": "daf05df4-45d9-4c60-811a-07bc88cacf0c",
+ "pk": "52ef38e2-e71e-4bcd-aa26-2eaad0d682ef",
"fields": {
- "worker": "25bfe0e8-7c7c-491e-b9b4-73498c4b5eb7",
- "revision": "2b024a21-5655-45b7-9fe0-d27b58a6a2d2",
+ "worker": "03ca969c-2fee-4a4c-b2f7-c283fb72bf91",
+ "revision": "97306172-bca6-46a6-9789-4b084e09c008",
"version": null,
"configuration": {
"test": 42
},
"state": "available",
- "gpu_usage": "disabled",
+ "gpu_usage": "required",
"model_usage": "disabled",
- "docker_image": "a93301c3-bdac-4d06-ba22-b7066decd282",
+ "docker_image": "27166636-75bd-4296-adce-ba8fa09a36c5",
"docker_image_iid": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
@@ -373,18 +381,18 @@
},
{
"model": "process.workerversion",
- "pk": "e45fe700-2e3d-49c5-a40e-01e46009ac65",
+ "pk": "6fe5da3e-18f0-4c08-ab75-631c527534df",
"fields": {
- "worker": "fc99199f-0f3a-4c0d-a033-db7f151910b1",
- "revision": "2b024a21-5655-45b7-9fe0-d27b58a6a2d2",
+ "worker": "3d121605-190a-4faa-8085-e7c1bffc4297",
+ "revision": "97306172-bca6-46a6-9789-4b084e09c008",
"version": null,
"configuration": {
"test": 42
},
"state": "available",
"gpu_usage": "disabled",
- "model_usage": "required",
- "docker_image": "a93301c3-bdac-4d06-ba22-b7066decd282",
+ "model_usage": "disabled",
+ "docker_image": "27166636-75bd-4296-adce-ba8fa09a36c5",
"docker_image_iid": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
@@ -392,18 +400,18 @@
},
{
"model": "process.workerversion",
- "pk": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "pk": "c82df68d-f845-4e3d-a1de-4c84be97230a",
"fields": {
- "worker": "08ef0527-5af6-45c7-a84c-d9d69697e5ca",
- "revision": "2b024a21-5655-45b7-9fe0-d27b58a6a2d2",
- "version": null,
+ "worker": "6a9a5388-c047-4bbf-9c41-86fbc578fd92",
+ "revision": null,
+ "version": 1,
"configuration": {
- "test": 42
+ "custom": "value"
},
- "state": "available",
+ "state": "created",
"gpu_usage": "disabled",
"model_usage": "disabled",
- "docker_image": "a93301c3-bdac-4d06-ba22-b7066decd282",
+ "docker_image": null,
"docker_image_iid": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
@@ -411,10 +419,10 @@
},
{
"model": "process.workerrun",
- "pk": "4c989351-7f30-4bb0-8424-31f56b33f925",
+ "pk": "4f2c84cf-f641-4d79-a7f0-d21075e61632",
"fields": {
- "process": "7b0b9c75-bffc-42f5-a646-c81e8197d171",
- "version": "5bb8f594-dff8-4145-8634-9ba88ff8a61d",
+ "process": "6f8334b3-b89c-48ef-b69f-255a5c4f868d",
+ "version": "c82df68d-f845-4e3d-a1de-4c84be97230a",
"model_version": null,
"parents": "[]",
"configuration": null,
@@ -425,49 +433,49 @@
},
{
"model": "process.workerrun",
- "pk": "5e8879fd-3da3-4bb0-83db-044d97ca9f89",
+ "pk": "ca9e5c86-ac92-4f73-b170-fff5e6331888",
"fields": {
- "process": "8cec0605-06e6-4b8f-ba7d-3ec9f9f2a409",
- "version": "daf05df4-45d9-4c60-811a-07bc88cacf0c",
+ "process": "d9f1b26c-6aae-4e22-a9cc-d218a245794e",
+ "version": "6fe5da3e-18f0-4c08-ab75-631c527534df",
"model_version": null,
"parents": "[]",
"configuration": null,
- "summary": "Worker Document layout analyser @ daf05d",
+ "summary": "Worker Document layout analyser @ 6fe5da",
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
}
},
{
"model": "process.workerrun",
- "pk": "c7fb12ab-d10f-4091-95d2-141e5b249a95",
+ "pk": "b68c0c89-1642-4ac3-813d-fa0fe5fbe54d",
"fields": {
- "process": "b4b4b4af-c221-4c47-a784-cc1bac1a99b1",
- "version": "5bb8f594-dff8-4145-8634-9ba88ff8a61d",
+ "process": "d9f1b26c-6aae-4e22-a9cc-d218a245794e",
+ "version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"model_version": null,
- "parents": "[]",
+ "parents": "[\"ca9e5c86-ac92-4f73-b170-fff5e6331888\"]",
"configuration": null,
- "summary": "Worker Custom worker @ version 1",
+ "summary": "Worker Recognizer @ 38af22",
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
}
},
{
"model": "process.workerrun",
- "pk": "2d18ad3f-6e5e-4184-8518-dda1d384164e",
+ "pk": "b88f23f6-af5d-4e20-bc58-fcc4c9966c5f",
"fields": {
- "process": "8cec0605-06e6-4b8f-ba7d-3ec9f9f2a409",
- "version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "process": "55e92ea6-625a-4c30-9724-b1f327941cac",
+ "version": "c82df68d-f845-4e3d-a1de-4c84be97230a",
"model_version": null,
- "parents": "[\"5e8879fd-3da3-4bb0-83db-044d97ca9f89\"]",
+ "parents": "[]",
"configuration": null,
- "summary": "Worker Recognizer @ f22fde",
+ "summary": "Worker Custom worker @ version 1",
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z"
}
},
{
"model": "documents.corpus",
- "pk": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "pk": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
@@ -480,33 +488,21 @@
},
{
"model": "documents.elementtype",
- "pk": "16c614aa-d6d4-4f55-afdd-0fada214362a",
+ "pk": "5b656298-6b77-441d-89bc-944f83e90318",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "slug": "surface",
- "display_name": "Surface",
- "folder": false,
- "indexable": false,
- "color": "28b62c"
- }
-},
-{
- "model": "documents.elementtype",
- "pk": "1acf2b7e-3a52-40b4-b428-40b17992928f",
- "fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "slug": "act",
- "display_name": "Act",
- "folder": false,
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "slug": "volume",
+ "display_name": "Volume",
+ "folder": true,
"indexable": false,
"color": "28b62c"
}
},
{
"model": "documents.elementtype",
- "pk": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
+ "pk": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"slug": "word",
"display_name": "Word",
"folder": false,
@@ -516,23 +512,23 @@
},
{
"model": "documents.elementtype",
- "pk": "83f66e14-eeaf-44e5-8be2-f1eddaa1a549",
+ "pk": "ac394477-b189-4a8c-b61a-194e15c7ca75",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "slug": "volume",
- "display_name": "Volume",
- "folder": true,
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "slug": "surface",
+ "display_name": "Surface",
+ "folder": false,
"indexable": false,
"color": "28b62c"
}
},
{
"model": "documents.elementtype",
- "pk": "9a893dfa-75b2-4ecd-b8a4-0fa532faadfe",
+ "pk": "b377e013-b313-412a-bd95-58681f87857c",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "slug": "text_line",
- "display_name": "Line",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "slug": "act",
+ "display_name": "Act",
"folder": false,
"indexable": false,
"color": "28b62c"
@@ -540,9 +536,9 @@
},
{
"model": "documents.elementtype",
- "pk": "e2c64e0b-a931-4453-8d9f-e84876583f45",
+ "pk": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"slug": "page",
"display_name": "Page",
"folder": false,
@@ -551,280 +547,292 @@
}
},
{
- "model": "documents.elementpath",
- "pk": "065e9fcf-4b9e-4ed4-96ab-9a8b4dd7adbd",
+ "model": "documents.elementtype",
+ "pk": "d5db136a-308b-4719-8553-b9f1485dbd31",
"fields": {
- "element": "4405ebfe-b7db-43d1-8ccc-2f90443631c5",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 7
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "slug": "text_line",
+ "display_name": "Line",
+ "folder": false,
+ "indexable": false,
+ "color": "28b62c"
}
},
{
"model": "documents.elementpath",
- "pk": "196df0fb-7dd3-4c21-bec5-86c68f702c2b",
+ "pk": "003b020e-d3e9-4ab7-b5de-2bf54ceba1e0",
"fields": {
- "element": "d418351a-7ad2-4ee6-9cb4-d306cdd0843d",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"9d4abe00-9e52-4ddf-a89d-5f4a67b6433b\"]",
+ "element": "41d359f5-3f19-4f17-9cbb-83b68f124ed4",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"a7d90223-fc30-4409-8ae6-2505695867d4\"]",
"ordering": 2
}
},
{
"model": "documents.elementpath",
- "pk": "21c4a14a-77e7-4af1-a521-b1b49b95e190",
+ "pk": "077144c7-d29c-4230-b87e-81336ec32ff1",
"fields": {
- "element": "23d22c17-9dc2-4034-978c-5ef2b6eff576",
+ "element": "7aa6af77-3552-46ff-afdd-ecf210c2da44",
"path": "[]",
"ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "23b53148-bd39-4e84-b4f2-ea3f1f601a74",
+ "pk": "153359d1-24aa-4231-a101-0df86d3302fe",
"fields": {
- "element": "012cd391-bedc-4159-9be0-19d0b3c54f95",
- "path": "[]",
- "ordering": 0
+ "element": "a344a86c-c126-464b-927f-40a4583de0a2",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
+ "ordering": 4
}
},
{
"model": "documents.elementpath",
- "pk": "37399b33-366c-4c50-bdb1-93286fda8f39",
+ "pk": "22f15836-34e5-4eed-975c-4eaa44cae20f",
"fields": {
- "element": "e8b3cc6e-6518-42d5-928e-d14cb6cd26ac",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"5a2b2988-b759-4fad-8282-1aae9cf39a31\"]",
+ "element": "01c3a423-096e-412d-b559-b1d6bcc2f4bc",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"bef956e0-f9f1-43c6-9409-cc64588fd4c2\"]",
"ordering": 1
}
},
{
"model": "documents.elementpath",
- "pk": "48721e40-61d5-4860-895d-aea3f41282a3",
+ "pk": "25fff9a2-6747-4cfa-adab-29f1156c3926",
"fields": {
- "element": "15384754-31b5-43b1-b9e3-b39a07a893d6",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"9d4abe00-9e52-4ddf-a89d-5f4a67b6433b\"]",
- "ordering": 1
+ "element": "9ce76cf5-862d-4645-8469-2d9bca27c5e9",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"6ef0d61a-9fd2-417e-b8cf-a68cdc569f8d\"]",
+ "ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "65cf5360-7d04-40dd-b74b-a13b70932510",
+ "pk": "2f797330-815a-4b4f-b033-0210ce06f116",
"fields": {
- "element": "393274b6-d4c0-425f-8a4e-6c957d527c83",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"9d4abe00-9e52-4ddf-a89d-5f4a67b6433b\"]",
+ "element": "429edf14-afcc-46f9-b2a0-5c24fab6aa57",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"d01d0065-f45f-42c5-82c9-d7b3e9e98cd1\"]",
"ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "6cbc60e4-a853-4ea8-8f81-001309f1e486",
+ "pk": "479d4e56-c8a7-4275-90ff-7775d385a1af",
"fields": {
- "element": "69814c5b-6f75-4bcb-aba0-60d9e3093bc5",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"a22f1a4f-ee4e-44f4-8ea8-217f685b77d4\"]",
- "ordering": 0
+ "element": "02d9886a-65ee-4c39-b961-a28d2178df73",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"a7d90223-fc30-4409-8ae6-2505695867d4\"]",
+ "ordering": 1
}
},
{
"model": "documents.elementpath",
- "pk": "7142899c-fe90-4ca5-8ffb-0dcf1d2434a7",
+ "pk": "4b21fc9a-fd94-4567-a02c-2ddfb0a751d8",
"fields": {
- "element": "cbbbdf83-7832-4487-a76a-6eabb040df3d",
- "path": "[\"23d22c17-9dc2-4034-978c-5ef2b6eff576\"]",
- "ordering": 0
+ "element": "baccd487-e8fb-456a-9a2e-4fe0827d38bf",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
+ "ordering": 7
}
},
{
"model": "documents.elementpath",
- "pk": "7bbcace8-10d3-48a5-834f-b34a72b68b8d",
+ "pk": "554746d0-9765-490b-91ad-a25d0369c0d3",
"fields": {
- "element": "d7165395-0cbf-4070-bd46-ad050fb63d33",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"b1fef6ad-84b9-4590-82dc-7d5e94f51772\"]",
- "ordering": 0
+ "element": "794f4388-a512-49ad-abb8-6e9d234895f3",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"d01d0065-f45f-42c5-82c9-d7b3e9e98cd1\"]",
+ "ordering": 1
}
},
{
"model": "documents.elementpath",
- "pk": "7db41281-a9ea-4b19-a599-3147d6e29bdd",
+ "pk": "58f9ecc4-9b8f-4bf2-86bf-172fc8c76b9e",
"fields": {
- "element": "bd32400c-be2c-4fdf-bbb9-0166aad2e04b",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 3
+ "element": "f769df85-a9c6-49ae-a42a-32d01c8b0103",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"a344a86c-c126-464b-927f-40a4583de0a2\"]",
+ "ordering": 1
}
},
{
"model": "documents.elementpath",
- "pk": "858bb5c7-4981-4455-bfe6-ea5bbd7405e8",
+ "pk": "62c6e9ef-34e6-453d-9e84-da6e857797c7",
"fields": {
- "element": "b1fef6ad-84b9-4590-82dc-7d5e94f51772",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 1
+ "element": "5fc4e41b-a399-4794-84b4-6821bee6a996",
+ "path": "[\"7aa6af77-3552-46ff-afdd-ecf210c2da44\"]",
+ "ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "878c84b9-6983-481a-8117-7806647c50b0",
+ "pk": "649d5264-3184-4b2c-a452-c3575c28a352",
"fields": {
- "element": "f18b3232-3ef4-4ae0-a125-02dca0d859d6",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"5a2b2988-b759-4fad-8282-1aae9cf39a31\"]",
- "ordering": 3
+ "element": "d15b508f-b7f4-4521-9346-757c641c7f54",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"bef956e0-f9f1-43c6-9409-cc64588fd4c2\"]",
+ "ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "95a27f11-258f-41b8-9f40-dd7ebd26419c",
+ "pk": "67f05365-3912-4135-ad73-6c9642876da6",
"fields": {
- "element": "9e9da196-7011-4c87-be3c-21b580992c93",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 4
+ "element": "2606fa5a-4356-4646-a6b6-c0d9adced4c8",
+ "path": "[\"7aa6af77-3552-46ff-afdd-ecf210c2da44\"]",
+ "ordering": 2
}
},
{
"model": "documents.elementpath",
- "pk": "99989b98-d1a2-43e6-865c-ecf081147280",
+ "pk": "72332ada-b46e-473c-8681-eacbdcdd4e0c",
"fields": {
- "element": "22a1757a-1af7-4999-8e45-0428ea23cfed",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"b1fef6ad-84b9-4590-82dc-7d5e94f51772\"]",
- "ordering": 1
+ "element": "6ef0d61a-9fd2-417e-b8cf-a68cdc569f8d",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
+ "ordering": 6
+ }
+},
+{
+ "model": "documents.elementpath",
+ "pk": "96af6dbc-51df-4bd2-a67f-cb211d51a246",
+ "fields": {
+ "element": "649a8f7c-2db7-43cf-82a5-ca86206d9d46",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
+ "ordering": 5
}
},
{
"model": "documents.elementpath",
- "pk": "b1b1288b-90cc-43d6-8f67-8247776093cb",
+ "pk": "9bbe0819-eea7-48a5-9d4c-cbe33968412f",
"fields": {
- "element": "deb1ed5d-b954-4450-a4b4-edb0741efec8",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"9e9da196-7011-4c87-be3c-21b580992c93\"]",
+ "element": "bef956e0-f9f1-43c6-9409-cc64588fd4c2",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
"ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "b96f33dc-316f-43b3-8a4d-2d42232f470d",
+ "pk": "a2377e58-1a90-46a6-8d25-b805e8dd0621",
"fields": {
- "element": "ad6483f2-c90a-41ba-9dbd-9cd3dbf78146",
- "path": "[\"23d22c17-9dc2-4034-978c-5ef2b6eff576\"]",
- "ordering": 2
+ "element": "5012bb0c-0249-4507-813e-879682b9321e",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"a7d90223-fc30-4409-8ae6-2505695867d4\"]",
+ "ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "bce997cc-26c1-4175-9ff8-5fc36ab6f32d",
+ "pk": "b7765894-b58d-44d9-8293-3757dbf846cb",
"fields": {
- "element": "d6673022-bbc1-49a6-9d15-3eb68c7f7616",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 6
+ "element": "bb6a86ce-c3a9-47e0-8f6e-9dc9eb067d98",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"a344a86c-c126-464b-927f-40a4583de0a2\"]",
+ "ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "c3ee3805-4418-4fdb-885b-19f3e8044ca4",
+ "pk": "b7a1b98c-f79e-4e72-ad2c-f9c3cb0525bd",
"fields": {
- "element": "05ca5b27-06c8-4337-8548-9de459d39024",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"b1fef6ad-84b9-4590-82dc-7d5e94f51772\"]",
+ "element": "a6ec2e2c-7f41-4fc8-aa05-c3d3a4ffd44d",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"bef956e0-f9f1-43c6-9409-cc64588fd4c2\"]",
"ordering": 2
}
},
{
"model": "documents.elementpath",
- "pk": "c7d8d602-d24b-4d57-a370-3eed48992776",
+ "pk": "b9700ac6-7339-437c-8a4a-a0154d4d490e",
"fields": {
- "element": "5beef449-d994-407d-a881-d4a7d0e14ad7",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"5a2b2988-b759-4fad-8282-1aae9cf39a31\"]",
+ "element": "1a2f95f4-c69c-4d3e-babc-0cc61ea04b8b",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"baccd487-e8fb-456a-9a2e-4fe0827d38bf\"]",
"ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "cbc0702b-4cc2-4b85-a310-1a288bbec5ba",
+ "pk": "bc200b4d-4eba-43f5-a800-573609152cbd",
"fields": {
- "element": "bd078be2-bff4-4c8b-ae78-241f1e4b144b",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"4405ebfe-b7db-43d1-8ccc-2f90443631c5\"]",
+ "element": "cc07dfc7-811c-4550-b2d9-b309f6c282ff",
+ "path": "[]",
"ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "dac53794-ecac-486b-b9eb-60788dbf6a66",
+ "pk": "c70833cc-31fd-48a4-9d87-7af6b8d78271",
"fields": {
- "element": "5a2b2988-b759-4fad-8282-1aae9cf39a31",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 0
+ "element": "cc0ec6c0-8eae-44ca-8179-7417456b8229",
+ "path": "[\"7aa6af77-3552-46ff-afdd-ecf210c2da44\"]",
+ "ordering": 1
}
},
{
"model": "documents.elementpath",
- "pk": "db6fbd1f-d5ce-4b16-bc6b-3c96ae7234ca",
+ "pk": "cb05e758-c987-412d-b173-620b60e54bed",
"fields": {
- "element": "1a5805d2-da50-4fc3-916f-92402e479e6e",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"5a2b2988-b759-4fad-8282-1aae9cf39a31\"]",
- "ordering": 2
+ "element": "447fa573-bf5b-4683-ab69-b7dcdc57c1f5",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"649a8f7c-2db7-43cf-82a5-ca86206d9d46\"]",
+ "ordering": 0
}
},
{
"model": "documents.elementpath",
- "pk": "ddcb97c4-a28f-4936-938f-99d6fd5be54e",
+ "pk": "d5174202-d76c-43e0-bb09-a0bfd9046361",
"fields": {
- "element": "a22f1a4f-ee4e-44f4-8ea8-217f685b77d4",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 5
+ "element": "b13c0dc4-9e1d-4903-ba42-efff19ab32c2",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"bef956e0-f9f1-43c6-9409-cc64588fd4c2\"]",
+ "ordering": 3
}
},
{
"model": "documents.elementpath",
- "pk": "e40b2625-b378-4a15-96aa-401a6970f5e8",
+ "pk": "d5abdd9e-5dca-49a6-a0b7-fb9d20925c61",
"fields": {
- "element": "96f1d3f6-c65a-44bd-827f-d2569aecc629",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"bd32400c-be2c-4fdf-bbb9-0166aad2e04b\"]",
- "ordering": 0
+ "element": "a7d90223-fc30-4409-8ae6-2505695867d4",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
+ "ordering": 2
}
},
{
"model": "documents.elementpath",
- "pk": "e87ca8aa-d0f3-4beb-8af6-438b4f3d543a",
+ "pk": "dbcb356a-d7da-4f81-99c4-86dbad2d4420",
"fields": {
- "element": "29f17e00-1d4a-46d8-91c3-db65667c2503",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"d6673022-bbc1-49a6-9d15-3eb68c7f7616\"]",
- "ordering": 0
+ "element": "6c0b88e8-e83f-4781-ac4b-e46944e4e2f6",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
+ "ordering": 3
}
},
{
"model": "documents.elementpath",
- "pk": "eba4b8e5-37fa-4d92-9136-fad469f1cc64",
+ "pk": "e64a336d-20c9-4c8b-9fb7-0632aa1dd7ac",
"fields": {
- "element": "28eb7ab5-b1da-46d2-bb94-f461570a4918",
- "path": "[\"23d22c17-9dc2-4034-978c-5ef2b6eff576\"]",
- "ordering": 1
+ "element": "7311f153-7a61-4fef-875b-ed36a2537fce",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"d01d0065-f45f-42c5-82c9-d7b3e9e98cd1\"]",
+ "ordering": 2
}
},
{
"model": "documents.elementpath",
- "pk": "f1bb6429-7ddb-4bf5-8dde-fa7b4451ae7b",
+ "pk": "e9518a0a-0210-42d8-8f3e-fa7b25c64f49",
"fields": {
- "element": "80bbf4bf-5840-400d-a450-b777cd7508aa",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\", \"9e9da196-7011-4c87-be3c-21b580992c93\"]",
+ "element": "d01d0065-f45f-42c5-82c9-d7b3e9e98cd1",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\"]",
"ordering": 1
}
},
{
"model": "documents.elementpath",
- "pk": "f40e4025-286a-4758-93b9-660672e58f85",
+ "pk": "f3d1dfb5-ab42-4a2b-ac74-ad914dd42985",
"fields": {
- "element": "9d4abe00-9e52-4ddf-a89d-5f4a67b6433b",
- "path": "[\"012cd391-bedc-4159-9be0-19d0b3c54f95\"]",
- "ordering": 2
+ "element": "3a4f3fd5-ca40-4dad-8317-532565ec3def",
+ "path": "[\"cc07dfc7-811c-4550-b2d9-b309f6c282ff\", \"6c0b88e8-e83f-4781-ac4b-e46944e4e2f6\"]",
+ "ordering": 0
}
},
{
"model": "documents.element",
- "pk": "012cd391-bedc-4159-9be0-19d0b3c54f95",
+ "pk": "01c3a423-096e-412d-b559-b1d6bcc2f4bc",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "83f66e14-eeaf-44e5-8be2-f1eddaa1a549",
- "name": "Volume 1",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "ROY",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": null,
- "polygon": null,
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -832,18 +840,18 @@
},
{
"model": "documents.element",
- "pk": "05ca5b27-06c8-4337-8548-9de459d39024",
+ "pk": "02d9886a-65ee-4c39-b961-a28d2178df73",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "DATUM",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "ROY",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "749d041a-c431-4617-85ff-a0628f38737d",
- "polygon": "LINEARRING (700 700, 700 800, 800 800, 800 700, 700 700)",
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
+ "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -851,18 +859,18 @@
},
{
"model": "documents.element",
- "pk": "15384754-31b5-43b1-b9e3-b39a07a893d6",
+ "pk": "1a2f95f4-c69c-4d3e-babc-0cc61ea04b8b",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "ROY",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "ac394477-b189-4a8c-b61a-194e15c7ca75",
+ "name": "Surface F",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
- "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
+ "polygon": "LINEARRING (600 600, 600 1000, 1000 1000, 1000 600, 600 600)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -870,18 +878,18 @@
},
{
"model": "documents.element",
- "pk": "1a5805d2-da50-4fc3-916f-92402e479e6e",
+ "pk": "2606fa5a-4356-4646-a6b6-c0d9adced4c8",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "DATUM",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
+ "name": "Volume 2, page 2r",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (700 700, 700 800, 800 800, 800 700, 700 700)",
+ "image": "3a90a6c2-5c43-4af2-993e-b46091607be6",
+ "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -889,18 +897,18 @@
},
{
"model": "documents.element",
- "pk": "22a1757a-1af7-4999-8e45-0428ea23cfed",
+ "pk": "3a4f3fd5-ca40-4dad-8317-532565ec3def",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "ROY",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "ac394477-b189-4a8c-b61a-194e15c7ca75",
+ "name": "Surface A",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "749d041a-c431-4617-85ff-a0628f38737d",
- "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (0 0, 0 600, 600 600, 600 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -908,18 +916,18 @@
},
{
"model": "documents.element",
- "pk": "23d22c17-9dc2-4034-978c-5ef2b6eff576",
+ "pk": "41d359f5-3f19-4f17-9cbb-83b68f124ed4",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "83f66e14-eeaf-44e5-8be2-f1eddaa1a549",
- "name": "Volume 2",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "DATUM",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": null,
- "polygon": null,
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
+ "polygon": "LINEARRING (700 700, 700 800, 800 800, 800 700, 700 700)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -927,18 +935,18 @@
},
{
"model": "documents.element",
- "pk": "28eb7ab5-b1da-46d2-bb94-f461570a4918",
+ "pk": "429edf14-afcc-46f9-b2a0-5c24fab6aa57",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "e2c64e0b-a931-4453-8d9f-e84876583f45",
- "name": "Volume 2, page 1v",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "PARIS",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "3ce03720-068d-45f0-9237-df2d744ef953",
- "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
+ "image": "333f1f5d-eb17-40a9-8401-d2303a978d5d",
+ "polygon": "LINEARRING (100 100, 100 200, 200 200, 200 100, 100 100)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -946,18 +954,18 @@
},
{
"model": "documents.element",
- "pk": "29f17e00-1d4a-46d8-91c3-db65667c2503",
+ "pk": "447fa573-bf5b-4683-ab69-b7dcdc57c1f5",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "16c614aa-d6d4-4f55-afdd-0fada214362a",
- "name": "Surface E",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "ac394477-b189-4a8c-b61a-194e15c7ca75",
+ "name": "Surface D",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
- "polygon": "LINEARRING (300 300, 300 600, 600 600, 600 300, 300 300)",
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
+ "polygon": "LINEARRING (0 0, 0 300, 300 300, 300 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -965,17 +973,17 @@
},
{
"model": "documents.element",
- "pk": "393274b6-d4c0-425f-8a4e-6c957d527c83",
+ "pk": "5012bb0c-0249-4507-813e-879682b9321e",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
"name": "PARIS",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
"polygon": "LINEARRING (100 100, 100 200, 200 200, 200 100, 100 100)",
"rotation_angle": 0,
"mirrored": false,
@@ -984,18 +992,18 @@
},
{
"model": "documents.element",
- "pk": "4405ebfe-b7db-43d1-8ccc-2f90443631c5",
+ "pk": "5fc4e41b-a399-4794-84b4-6821bee6a996",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "1acf2b7e-3a52-40b4-b428-40b17992928f",
- "name": "Act 5",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
+ "name": "Volume 2, page 1r",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": null,
- "polygon": null,
+ "image": "65351fbb-f6ce-429b-8615-3782bd4e1192",
+ "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1003,18 +1011,18 @@
},
{
"model": "documents.element",
- "pk": "5a2b2988-b759-4fad-8282-1aae9cf39a31",
+ "pk": "649a8f7c-2db7-43cf-82a5-ca86206d9d46",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "e2c64e0b-a931-4453-8d9f-e84876583f45",
- "name": "Volume 1, page 1r",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "b377e013-b313-412a-bd95-58681f87857c",
+ "name": "Act 3",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
+ "image": null,
+ "polygon": null,
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1022,18 +1030,18 @@
},
{
"model": "documents.element",
- "pk": "5beef449-d994-407d-a881-d4a7d0e14ad7",
+ "pk": "6c0b88e8-e83f-4781-ac4b-e46944e4e2f6",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "PARIS",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "b377e013-b313-412a-bd95-58681f87857c",
+ "name": "Act 1",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (100 100, 100 200, 200 200, 200 100, 100 100)",
+ "image": null,
+ "polygon": null,
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1041,18 +1049,18 @@
},
{
"model": "documents.element",
- "pk": "69814c5b-6f75-4bcb-aba0-60d9e3093bc5",
+ "pk": "6ef0d61a-9fd2-417e-b8cf-a68cdc569f8d",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "16c614aa-d6d4-4f55-afdd-0fada214362a",
- "name": "Surface D",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "b377e013-b313-412a-bd95-58681f87857c",
+ "name": "Act 4",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
- "polygon": "LINEARRING (0 0, 0 300, 300 300, 300 0, 0 0)",
+ "image": null,
+ "polygon": null,
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1060,18 +1068,18 @@
},
{
"model": "documents.element",
- "pk": "80bbf4bf-5840-400d-a450-b777cd7508aa",
+ "pk": "7311f153-7a61-4fef-875b-ed36a2537fce",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "16c614aa-d6d4-4f55-afdd-0fada214362a",
- "name": "Surface C",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "DATUM",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "749d041a-c431-4617-85ff-a0628f38737d",
- "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
+ "image": "333f1f5d-eb17-40a9-8401-d2303a978d5d",
+ "polygon": "LINEARRING (700 700, 700 800, 800 800, 800 700, 700 700)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1079,18 +1087,18 @@
},
{
"model": "documents.element",
- "pk": "96f1d3f6-c65a-44bd-827f-d2569aecc629",
+ "pk": "794f4388-a512-49ad-abb8-6e9d234895f3",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "16c614aa-d6d4-4f55-afdd-0fada214362a",
- "name": "Surface A",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "ROY",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (0 0, 0 600, 600 600, 600 0, 0 0)",
+ "image": "333f1f5d-eb17-40a9-8401-d2303a978d5d",
+ "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1098,18 +1106,18 @@
},
{
"model": "documents.element",
- "pk": "9d4abe00-9e52-4ddf-a89d-5f4a67b6433b",
+ "pk": "7aa6af77-3552-46ff-afdd-ecf210c2da44",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "e2c64e0b-a931-4453-8d9f-e84876583f45",
- "name": "Volume 1, page 2r",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "5b656298-6b77-441d-89bc-944f83e90318",
+ "name": "Volume 2",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
- "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
+ "image": null,
+ "polygon": null,
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1117,18 +1125,18 @@
},
{
"model": "documents.element",
- "pk": "9e9da196-7011-4c87-be3c-21b580992c93",
+ "pk": "9ce76cf5-862d-4645-8469-2d9bca27c5e9",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "1acf2b7e-3a52-40b4-b428-40b17992928f",
- "name": "Act 2",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "ac394477-b189-4a8c-b61a-194e15c7ca75",
+ "name": "Surface E",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": null,
- "polygon": null,
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
+ "polygon": "LINEARRING (300 300, 300 600, 600 600, 600 300, 300 300)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1136,13 +1144,13 @@
},
{
"model": "documents.element",
- "pk": "a22f1a4f-ee4e-44f4-8ea8-217f685b77d4",
+ "pk": "a344a86c-c126-464b-927f-40a4583de0a2",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "1acf2b7e-3a52-40b4-b428-40b17992928f",
- "name": "Act 3",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "b377e013-b313-412a-bd95-58681f87857c",
+ "name": "Act 2",
"creator": null,
"worker_version": null,
"worker_run": null,
@@ -1155,18 +1163,18 @@
},
{
"model": "documents.element",
- "pk": "ad6483f2-c90a-41ba-9dbd-9cd3dbf78146",
+ "pk": "a6ec2e2c-7f41-4fc8-aa05-c3d3a4ffd44d",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "e2c64e0b-a931-4453-8d9f-e84876583f45",
- "name": "Volume 2, page 2r",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "DATUM",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "7841d614-be5e-4ff6-82dd-4089cc644a19",
- "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (700 700, 700 800, 800 800, 800 700, 700 700)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1174,17 +1182,17 @@
},
{
"model": "documents.element",
- "pk": "b1fef6ad-84b9-4590-82dc-7d5e94f51772",
+ "pk": "a7d90223-fc30-4409-8ae6-2505695867d4",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "e2c64e0b-a931-4453-8d9f-e84876583f45",
- "name": "Volume 1, page 1v",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
+ "name": "Volume 1, page 2r",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "749d041a-c431-4617-85ff-a0628f38737d",
+ "image": "c826c86a-e5d4-4de8-997b-31b65caf7011",
"polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
@@ -1193,18 +1201,18 @@
},
{
"model": "documents.element",
- "pk": "bd078be2-bff4-4c8b-ae78-241f1e4b144b",
+ "pk": "b13c0dc4-9e1d-4903-ba42-efff19ab32c2",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "16c614aa-d6d4-4f55-afdd-0fada214362a",
- "name": "Surface F",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "d5db136a-308b-4719-8553-b9f1485dbd31",
+ "name": "Text line",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
- "polygon": "LINEARRING (600 600, 600 1000, 1000 1000, 1000 600, 600 600)",
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1212,13 +1220,13 @@
},
{
"model": "documents.element",
- "pk": "bd32400c-be2c-4fdf-bbb9-0166aad2e04b",
+ "pk": "baccd487-e8fb-456a-9a2e-4fe0827d38bf",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "1acf2b7e-3a52-40b4-b428-40b17992928f",
- "name": "Act 1",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "b377e013-b313-412a-bd95-58681f87857c",
+ "name": "Act 5",
"creator": null,
"worker_version": null,
"worker_run": null,
@@ -1231,18 +1239,18 @@
},
{
"model": "documents.element",
- "pk": "cbbbdf83-7832-4487-a76a-6eabb040df3d",
+ "pk": "bb6a86ce-c3a9-47e0-8f6e-9dc9eb067d98",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "e2c64e0b-a931-4453-8d9f-e84876583f45",
- "name": "Volume 2, page 1r",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "ac394477-b189-4a8c-b61a-194e15c7ca75",
+ "name": "Surface B",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "2ee6f6af-ee3e-4ea8-86b6-d4bb0e4514ee",
- "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (600 600, 600 1000, 1000 1000, 1000 600, 600 600)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1250,18 +1258,18 @@
},
{
"model": "documents.element",
- "pk": "d418351a-7ad2-4ee6-9cb4-d306cdd0843d",
+ "pk": "bef956e0-f9f1-43c6-9409-cc64588fd4c2",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "DATUM",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
+ "name": "Volume 1, page 1r",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "9f324936-0561-432a-8985-103b240e55ed",
- "polygon": "LINEARRING (700 700, 700 800, 800 800, 800 700, 700 700)",
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1269,13 +1277,13 @@
},
{
"model": "documents.element",
- "pk": "d6673022-bbc1-49a6-9d15-3eb68c7f7616",
+ "pk": "cc07dfc7-811c-4550-b2d9-b309f6c282ff",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "1acf2b7e-3a52-40b4-b428-40b17992928f",
- "name": "Act 4",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "5b656298-6b77-441d-89bc-944f83e90318",
+ "name": "Volume 1",
"creator": null,
"worker_version": null,
"worker_run": null,
@@ -1288,18 +1296,18 @@
},
{
"model": "documents.element",
- "pk": "d7165395-0cbf-4070-bd46-ad050fb63d33",
+ "pk": "cc0ec6c0-8eae-44ca-8179-7417456b8229",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "PARIS",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
+ "name": "Volume 2, page 1v",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "749d041a-c431-4617-85ff-a0628f38737d",
- "polygon": "LINEARRING (100 100, 100 200, 200 200, 200 100, 100 100)",
+ "image": "9f7b7678-defd-4ce7-be9b-ed5b04da3c04",
+ "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1307,18 +1315,18 @@
},
{
"model": "documents.element",
- "pk": "deb1ed5d-b954-4450-a4b4-edb0741efec8",
+ "pk": "d01d0065-f45f-42c5-82c9-d7b3e9e98cd1",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "16c614aa-d6d4-4f55-afdd-0fada214362a",
- "name": "Surface B",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "c0686e6f-f7ee-47a6-91ef-276dc1c74604",
+ "name": "Volume 1, page 1v",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (600 600, 600 1000, 1000 1000, 1000 600, 600 600)",
+ "image": "333f1f5d-eb17-40a9-8401-d2303a978d5d",
+ "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1326,18 +1334,18 @@
},
{
"model": "documents.element",
- "pk": "e8b3cc6e-6518-42d5-928e-d14cb6cd26ac",
+ "pk": "d15b508f-b7f4-4521-9346-757c641c7f54",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "5e0a2e15-4493-4d27-a656-29396f01e9f6",
- "name": "ROY",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "a7a4d3cf-504a-4092-8828-5a19e3d31138",
+ "name": "PARIS",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
+ "image": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
+ "polygon": "LINEARRING (100 100, 100 200, 200 200, 200 100, 100 100)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1345,18 +1353,18 @@
},
{
"model": "documents.element",
- "pk": "f18b3232-3ef4-4ae0-a125-02dca0d859d6",
+ "pk": "f769df85-a9c6-49ae-a42a-32d01c8b0103",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "9a893dfa-75b2-4ecd-b8a4-0fa532faadfe",
- "name": "Text line",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "ac394477-b189-4a8c-b61a-194e15c7ca75",
+ "name": "Surface C",
"creator": null,
"worker_version": null,
"worker_run": null,
- "image": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
- "polygon": "LINEARRING (400 400, 400 500, 500 500, 500 400, 400 400)",
+ "image": "333f1f5d-eb17-40a9-8401-d2303a978d5d",
+ "polygon": "LINEARRING (0 0, 0 1000, 1000 1000, 1000 0, 0 0)",
"rotation_angle": 0,
"mirrored": false,
"confidence": null
@@ -1364,91 +1372,91 @@
},
{
"model": "documents.entitytype",
- "pk": "311d11f1-9a0d-42e8-b261-7f921d325a22",
+ "pk": "389cf24a-71b8-4d59-ba20-1bead9c19100",
"fields": {
- "name": "location",
+ "name": "organization",
"color": "ff0000",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa"
}
},
{
"model": "documents.entitytype",
- "pk": "7e7f01f6-fea3-454a-870f-1ef8a4416512",
+ "pk": "68cd175e-5519-469b-8795-4def01f68486",
"fields": {
- "name": "date",
+ "name": "person",
"color": "ff0000",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa"
}
},
{
"model": "documents.entitytype",
- "pk": "bbaaa893-ad44-45a0-aa03-975c8787ade1",
+ "pk": "79af3b80-5800-470e-9d50-76405371b35a",
"fields": {
- "name": "number",
+ "name": "location",
"color": "ff0000",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa"
}
},
{
"model": "documents.entitytype",
- "pk": "c0537bd2-7cc8-4612-abba-839869668a8c",
+ "pk": "a59ccd99-2b4f-4861-8480-e1d4481cd0d4",
"fields": {
- "name": "organization",
+ "name": "date",
"color": "ff0000",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa"
}
},
{
"model": "documents.entitytype",
- "pk": "d352ac6e-68a2-415b-898f-e234e58fb9f4",
+ "pk": "c4211cec-2f93-4e15-8e20-fb6597327b3c",
"fields": {
- "name": "person",
+ "name": "number",
"color": "ff0000",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa"
}
},
{
"model": "documents.transcription",
- "pk": "081a5c4f-7af4-4705-aa97-aeff028176dd",
+ "pk": "23e55bee-fae7-433f-a1ce-4ff250f8a985",
"fields": {
- "element": "5a2b2988-b759-4fad-8282-1aae9cf39a31",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "d15b508f-b7f4-4521-9346-757c641c7f54",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "Lorem ipsum dolor sit amet",
+ "text": "PARIS",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "162d8843-380a-4e28-9132-642fa8639018",
+ "pk": "2cb7dd1f-8189-4b80-974f-051e089feb42",
"fields": {
- "element": "393274b6-d4c0-425f-8a4e-6c957d527c83",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "41d359f5-3f19-4f17-9cbb-83b68f124ed4",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "PARIS",
+ "text": "DATUM",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "23641b79-24ef-4931-8cb4-57dc8c3d71d8",
+ "pk": "4fe2f9a4-927b-4d0e-a094-eacfad911931",
"fields": {
- "element": "e8b3cc6e-6518-42d5-928e-d14cb6cd26ac",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "5012bb0c-0249-4507-813e-879682b9321e",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "ROY",
+ "text": "PARIS",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "39d8309a-5ee5-4654-be82-c863c4c3719c",
+ "pk": "65f20dc0-3ec0-46b5-b539-fa1c22055f9e",
"fields": {
- "element": "05ca5b27-06c8-4337-8548-9de459d39024",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "7311f153-7a61-4fef-875b-ed36a2537fce",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
"text": "DATUM",
"orientation": "horizontal-lr",
@@ -1457,10 +1465,10 @@
},
{
"model": "documents.transcription",
- "pk": "4bb4012c-53b2-41be-ac0c-9d7b5067dc25",
+ "pk": "67d43d8f-0123-4d17-b496-527660f824d5",
"fields": {
- "element": "15384754-31b5-43b1-b9e3-b39a07a893d6",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "01c3a423-096e-412d-b559-b1d6bcc2f4bc",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
"text": "ROY",
"orientation": "horizontal-lr",
@@ -1469,99 +1477,99 @@
},
{
"model": "documents.transcription",
- "pk": "62bd07d6-3521-408c-b01c-e4005677b906",
+ "pk": "6926bb8f-773f-4505-9b75-e93481a80198",
"fields": {
- "element": "1a5805d2-da50-4fc3-916f-92402e479e6e",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "bef956e0-f9f1-43c6-9409-cc64588fd4c2",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "DATUM",
+ "text": "Lorem ipsum dolor sit amet",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "764094d1-cef3-40d5-bdb9-849f802d9335",
+ "pk": "9cca8460-b694-4f75-be5b-c12f14f3e374",
"fields": {
- "element": "5beef449-d994-407d-a881-d4a7d0e14ad7",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "02d9886a-65ee-4c39-b961-a28d2178df73",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "PARIS",
+ "text": "ROY",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "aef2094a-b509-4865-bbad-698ab78984ec",
+ "pk": "b0707c31-eb72-45e2-8a66-0d45a4011a0d",
"fields": {
- "element": "22a1757a-1af7-4999-8e45-0428ea23cfed",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "a6ec2e2c-7f41-4fc8-aa05-c3d3a4ffd44d",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "ROY",
+ "text": "DATUM",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "c9bb2b39-7f18-4979-afb7-c544ee8e6f4b",
+ "pk": "cec50683-2c40-4a32-a7bd-332b3241e392",
"fields": {
- "element": "d418351a-7ad2-4ee6-9cb4-d306cdd0843d",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "429edf14-afcc-46f9-b2a0-5c24fab6aa57",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "DATUM",
+ "text": "PARIS",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.transcription",
- "pk": "dc25ce06-ac14-44c6-8ad6-aa1d3ab20551",
+ "pk": "fe7bbe41-a811-4f62-b85f-b449e922f00f",
"fields": {
- "element": "d7165395-0cbf-4070-bd46-ad050fb63d33",
- "worker_version": "f22fde5b-28db-459a-8257-cab7b58ef63d",
+ "element": "794f4388-a512-49ad-abb8-6e9d234895f3",
+ "worker_version": "38af2294-5e69-4c10-8daf-5bb1b58b182a",
"worker_run": null,
- "text": "PARIS",
+ "text": "ROY",
"orientation": "horizontal-lr",
"confidence": 1.0
}
},
{
"model": "documents.allowedmetadata",
- "pk": "3b1250cb-94e7-4f8f-90f8-469ad21e2bf8",
+ "pk": "844cba89-47f3-403f-a200-580fea379535",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "date",
- "name": "date"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "location",
+ "name": "location"
}
},
{
"model": "documents.allowedmetadata",
- "pk": "4f16502f-3813-43e0-871f-0984a1082672",
+ "pk": "e7707502-a7d2-4f1c-9c38-d5b95e3acc89",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
- "type": "location",
- "name": "location"
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
+ "type": "date",
+ "name": "date"
}
},
{
"model": "documents.allowedmetadata",
- "pk": "b965e5fc-e3ba-4232-a710-57b2c2144dda",
+ "pk": "ea5bd9a8-dd37-4d0f-b829-58b92d8cfdf5",
"fields": {
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"type": "text",
"name": "folio"
}
},
{
"model": "documents.metadata",
- "pk": "16f42e57-9495-406c-baae-51be7c97c544",
+ "pk": "2184f569-2343-4f99-b8b2-5865a8c79890",
"fields": {
- "element": "9d4abe00-9e52-4ddf-a89d-5f4a67b6433b",
+ "element": "d01d0065-f45f-42c5-82c9-d7b3e9e98cd1",
"name": "folio",
"type": "text",
- "value": "2r",
+ "value": "1v",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1569,12 +1577,12 @@
},
{
"model": "documents.metadata",
- "pk": "43f60c5e-2fbd-4ac5-ae01-5aa73ebd4e27",
+ "pk": "28ccc70d-88d4-4594-9b36-c4777dd75139",
"fields": {
- "element": "9e9da196-7011-4c87-be3c-21b580992c93",
- "name": "number",
+ "element": "bef956e0-f9f1-43c6-9409-cc64588fd4c2",
+ "name": "folio",
"type": "text",
- "value": "2",
+ "value": "1r",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1582,12 +1590,12 @@
},
{
"model": "documents.metadata",
- "pk": "49afc57d-3852-439d-aec3-dc9bae75d317",
+ "pk": "303b3216-7852-4d12-a7bc-792c228c819d",
"fields": {
- "element": "28eb7ab5-b1da-46d2-bb94-f461570a4918",
+ "element": "a7d90223-fc30-4409-8ae6-2505695867d4",
"name": "folio",
"type": "text",
- "value": "1v",
+ "value": "2r",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1595,12 +1603,12 @@
},
{
"model": "documents.metadata",
- "pk": "7172ad8f-274e-4762-aa71-4b67e831a085",
+ "pk": "4796be93-6d1c-4e88-85a5-bfa3fd4c2df5",
"fields": {
- "element": "cbbbdf83-7832-4487-a76a-6eabb040df3d",
- "name": "folio",
+ "element": "6ef0d61a-9fd2-417e-b8cf-a68cdc569f8d",
+ "name": "number",
"type": "text",
- "value": "1r",
+ "value": "4",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1608,9 +1616,9 @@
},
{
"model": "documents.metadata",
- "pk": "7feebc85-ae0c-4f84-a9c8-719c0334f365",
+ "pk": "50ed9eca-5b28-4ca3-8fe9-28829d245c41",
"fields": {
- "element": "5a2b2988-b759-4fad-8282-1aae9cf39a31",
+ "element": "5fc4e41b-a399-4794-84b4-6821bee6a996",
"name": "folio",
"type": "text",
"value": "1r",
@@ -1621,12 +1629,12 @@
},
{
"model": "documents.metadata",
- "pk": "995ffc3d-fd2b-4aa1-a0fd-bb06818d13c4",
+ "pk": "653bbae1-3470-47f3-b1e3-3369a5d0c2c6",
"fields": {
- "element": "d6673022-bbc1-49a6-9d15-3eb68c7f7616",
+ "element": "6c0b88e8-e83f-4781-ac4b-e46944e4e2f6",
"name": "number",
"type": "text",
- "value": "4",
+ "value": "1",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1634,12 +1642,12 @@
},
{
"model": "documents.metadata",
- "pk": "9a20fe23-faf6-4579-8aa3-164bf0ebc9db",
+ "pk": "821dadd3-2f8d-483c-92e0-ba88be0873aa",
"fields": {
- "element": "4405ebfe-b7db-43d1-8ccc-2f90443631c5",
- "name": "number",
+ "element": "2606fa5a-4356-4646-a6b6-c0d9adced4c8",
+ "name": "folio",
"type": "text",
- "value": "5",
+ "value": "2r",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1647,12 +1655,12 @@
},
{
"model": "documents.metadata",
- "pk": "9c7749d9-e519-44ec-8558-013f51bcff98",
+ "pk": "ad2dfca7-1056-41d0-8492-31aa1356b8b8",
"fields": {
- "element": "bd32400c-be2c-4fdf-bbb9-0166aad2e04b",
+ "element": "a344a86c-c126-464b-927f-40a4583de0a2",
"name": "number",
"type": "text",
- "value": "1",
+ "value": "2",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1660,9 +1668,9 @@
},
{
"model": "documents.metadata",
- "pk": "d9d22295-6f93-4741-b548-8fc253f0a5fb",
+ "pk": "b3d6b05d-5cf7-4ace-bca4-8b50cb6326c6",
"fields": {
- "element": "a22f1a4f-ee4e-44f4-8ea8-217f685b77d4",
+ "element": "649a8f7c-2db7-43cf-82a5-ca86206d9d46",
"name": "number",
"type": "text",
"value": "3",
@@ -1673,9 +1681,9 @@
},
{
"model": "documents.metadata",
- "pk": "de76c9c1-0151-4b06-afcd-b44621414de1",
+ "pk": "db3febc5-f623-45c0-966a-780d55f2be57",
"fields": {
- "element": "b1fef6ad-84b9-4590-82dc-7d5e94f51772",
+ "element": "cc0ec6c0-8eae-44ca-8179-7417456b8229",
"name": "folio",
"type": "text",
"value": "1v",
@@ -1686,12 +1694,12 @@
},
{
"model": "documents.metadata",
- "pk": "de83d6be-9880-48fd-bbe7-2d171c0893a2",
+ "pk": "e691d260-dd78-42e2-81e0-be893f3abd74",
"fields": {
- "element": "ad6483f2-c90a-41ba-9dbd-9cd3dbf78146",
- "name": "folio",
+ "element": "baccd487-e8fb-456a-9a2e-4fe0827d38bf",
+ "name": "number",
"type": "text",
- "value": "2r",
+ "value": "5",
"entity": null,
"worker_version": null,
"worker_run": null
@@ -1714,12 +1722,12 @@
},
{
"model": "images.image",
- "pk": "2ee6f6af-ee3e-4ea8-86b6-d4bb0e4514ee",
+ "pk": "333f1f5d-eb17-40a9-8401-d2303a978d5d",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"server": 1,
- "path": "img4",
+ "path": "img2",
"width": 1000,
"height": 1000,
"hash": null,
@@ -1728,12 +1736,12 @@
},
{
"model": "images.image",
- "pk": "3ce03720-068d-45f0-9237-df2d744ef953",
+ "pk": "3a90a6c2-5c43-4af2-993e-b46091607be6",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"server": 1,
- "path": "img5",
+ "path": "img6",
"width": 1000,
"height": 1000,
"hash": null,
@@ -1742,7 +1750,7 @@
},
{
"model": "images.image",
- "pk": "71b96a29-ad88-4906-abeb-fd4a50d61af1",
+ "pk": "3ac0c089-ac48-45dd-9b18-77c1499fbe0e",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
@@ -1756,12 +1764,12 @@
},
{
"model": "images.image",
- "pk": "749d041a-c431-4617-85ff-a0628f38737d",
+ "pk": "65351fbb-f6ce-429b-8615-3782bd4e1192",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"server": 1,
- "path": "img2",
+ "path": "img4",
"width": 1000,
"height": 1000,
"hash": null,
@@ -1770,12 +1778,12 @@
},
{
"model": "images.image",
- "pk": "7841d614-be5e-4ff6-82dd-4089cc644a19",
+ "pk": "9f7b7678-defd-4ce7-be9b-ed5b04da3c04",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"server": 1,
- "path": "img6",
+ "path": "img5",
"width": 1000,
"height": 1000,
"hash": null,
@@ -1784,7 +1792,7 @@
},
{
"model": "images.image",
- "pk": "9f324936-0561-432a-8985-103b240e55ed",
+ "pk": "c826c86a-e5d4-4de8-997b-31b65caf7011",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
@@ -1798,64 +1806,64 @@
},
{
"model": "users.right",
- "pk": "056e0095-c923-4ad6-9c82-cbc4fd03be58",
- "fields": {
- "user": 2,
- "group": null,
- "content_type": 35,
- "content_id": "735c70c6-bb6f-405e-8126-c01efd6ad747",
- "level": 100
- }
-},
-{
- "model": "users.right",
- "pk": "1b17d60e-bbc1-4116-bb4b-bb0cd2a65b0e",
+ "pk": "6aeee47f-82fc-4f8c-ba67-0950001f9165",
"fields": {
"user": 4,
"group": null,
"content_type": 35,
- "content_id": "735c70c6-bb6f-405e-8126-c01efd6ad747",
+ "content_id": "154b84fc-9b5e-4224-8198-369424a6044f",
"level": 10
}
},
{
"model": "users.right",
- "pk": "6b9658f5-5a20-4268-8d84-80f0a0adb551",
+ "pk": "83388f8c-ab23-4a8f-a93a-5bcf1b1737ed",
"fields": {
"user": 2,
"group": null,
"content_type": 12,
- "content_id": "409b6859-63b9-41f2-8449-ba737bca6624",
+ "content_id": "d4639943-3bc5-4839-ba11-bac03818050c",
"level": 10
}
},
{
"model": "users.right",
- "pk": "6bc35837-48e5-4bb5-bb41-642c5063b681",
+ "pk": "8422a688-f89c-450c-9f50-1aa390f8f679",
"fields": {
- "user": 3,
+ "user": 2,
"group": null,
"content_type": 35,
- "content_id": "735c70c6-bb6f-405e-8126-c01efd6ad747",
- "level": 50
+ "content_id": "154b84fc-9b5e-4224-8198-369424a6044f",
+ "level": 100
}
},
{
"model": "users.right",
- "pk": "cf11fab0-8fc9-41b5-9a96-6464cc1b5581",
+ "pk": "94d056b0-9e9a-41b5-8287-445947594902",
"fields": {
"user": 2,
"group": null,
"content_type": 20,
- "content_id": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "content_id": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"level": 100
}
},
+{
+ "model": "users.right",
+ "pk": "ee07cbba-7ca5-4a6d-ac27-b782c25d294e",
+ "fields": {
+ "user": 3,
+ "group": null,
+ "content_type": 35,
+ "content_id": "154b84fc-9b5e-4224-8198-369424a6044f",
+ "level": 50
+ }
+},
{
"model": "users.user",
"pk": 1,
"fields": {
- "password": "pbkdf2_sha256$390000$ZuasXMuh9i4eX0Ou9Aq1bL$QHVkSsvmF0+j2920P9h8CdsIDM8tK8/Qnv5m1Bvqu8s=",
+ "password": "pbkdf2_sha256$390000$R2D2dMVrU1brlgQ1SF0ww8$wbS/HqbXLiXpwanPA0BuF2I3DzULtf6iAFgOv0if9es=",
"last_login": null,
"email": "root@root.fr",
"display_name": "Admin",
@@ -1870,7 +1878,7 @@
"model": "users.user",
"pk": 2,
"fields": {
- "password": "pbkdf2_sha256$390000$dtjfH2x1Y4Ux9pYBLOeuxJ$61IjrNARDyLvmRE/9jdfbc5jrcJBtv4hlGeNJJ4KV/o=",
+ "password": "pbkdf2_sha256$390000$IPXLTmkcsCErqMmLpDQ1uG$5sNFF32ArLCA6dVY/jQ51sdtYomLFiyxsEtsLJz5QMo=",
"last_login": null,
"email": "user@user.fr",
"display_name": "Test user",
@@ -1913,26 +1921,13 @@
},
{
"model": "users.group",
- "pk": "735c70c6-bb6f-405e-8126-c01efd6ad747",
+ "pk": "154b84fc-9b5e-4224-8198-369424a6044f",
"fields": {
"name": "User group",
"public": false,
"use_in_new_project": false
}
},
-{
- "model": "users.oauthcredentials",
- "pk": "217e2e72-f917-46de-9760-0e140bec08eb",
- "fields": {
- "user": 2,
- "provider_url": "https://somewhere",
- "status": "created",
- "token": "oauth-token",
- "refresh_token": "refresh-token",
- "expiry": "2100-12-31T23:59:59.999Z",
- "account_name": null
- }
-},
{
"model": "auth.permission",
"pk": 1,
@@ -3205,806 +3200,770 @@
{
"model": "auth.permission",
"pk": 142,
- "fields": {
- "name": "Can add OAuth credentials",
- "content_type": 37,
- "codename": "add_oauthcredentials"
- }
-},
-{
- "model": "auth.permission",
- "pk": 143,
- "fields": {
- "name": "Can change OAuth credentials",
- "content_type": 37,
- "codename": "change_oauthcredentials"
- }
-},
-{
- "model": "auth.permission",
- "pk": 144,
- "fields": {
- "name": "Can delete OAuth credentials",
- "content_type": 37,
- "codename": "delete_oauthcredentials"
- }
-},
-{
- "model": "auth.permission",
- "pk": 145,
- "fields": {
- "name": "Can view OAuth credentials",
- "content_type": 37,
- "codename": "view_oauthcredentials"
- }
-},
-{
- "model": "auth.permission",
- "pk": 146,
"fields": {
"name": "Can add user scope",
- "content_type": 38,
+ "content_type": 37,
"codename": "add_userscope"
}
},
{
"model": "auth.permission",
- "pk": 147,
+ "pk": 143,
"fields": {
"name": "Can change user scope",
- "content_type": 38,
+ "content_type": 37,
"codename": "change_userscope"
}
},
{
"model": "auth.permission",
- "pk": 148,
+ "pk": 144,
"fields": {
"name": "Can delete user scope",
- "content_type": 38,
+ "content_type": 37,
"codename": "delete_userscope"
}
},
{
"model": "auth.permission",
- "pk": 149,
+ "pk": 145,
"fields": {
"name": "Can view user scope",
- "content_type": 38,
+ "content_type": 37,
"codename": "view_userscope"
}
},
{
"model": "auth.permission",
- "pk": 150,
+ "pk": 146,
"fields": {
"name": "Can add corpus worker version",
- "content_type": 39,
+ "content_type": 38,
"codename": "add_corpusworkerversion"
}
},
{
"model": "auth.permission",
- "pk": 151,
+ "pk": 147,
"fields": {
"name": "Can change corpus worker version",
- "content_type": 39,
+ "content_type": 38,
"codename": "change_corpusworkerversion"
}
},
{
"model": "auth.permission",
- "pk": 152,
+ "pk": 148,
"fields": {
"name": "Can delete corpus worker version",
- "content_type": 39,
+ "content_type": 38,
"codename": "delete_corpusworkerversion"
}
},
{
"model": "auth.permission",
- "pk": 153,
+ "pk": 149,
"fields": {
"name": "Can view corpus worker version",
- "content_type": 39,
+ "content_type": 38,
"codename": "view_corpusworkerversion"
}
},
{
"model": "auth.permission",
- "pk": 154,
+ "pk": 150,
"fields": {
"name": "Can add data file",
- "content_type": 40,
+ "content_type": 39,
"codename": "add_datafile"
}
},
{
"model": "auth.permission",
- "pk": 155,
+ "pk": 151,
"fields": {
"name": "Can change data file",
- "content_type": 40,
+ "content_type": 39,
"codename": "change_datafile"
}
},
{
"model": "auth.permission",
- "pk": 156,
+ "pk": 152,
"fields": {
"name": "Can delete data file",
- "content_type": 40,
+ "content_type": 39,
"codename": "delete_datafile"
}
},
{
"model": "auth.permission",
- "pk": 157,
+ "pk": 153,
"fields": {
"name": "Can view data file",
- "content_type": 40,
+ "content_type": 39,
"codename": "view_datafile"
}
},
{
"model": "auth.permission",
- "pk": 158,
+ "pk": 154,
"fields": {
"name": "Can add git ref",
- "content_type": 41,
+ "content_type": 40,
"codename": "add_gitref"
}
},
{
"model": "auth.permission",
- "pk": 159,
+ "pk": 155,
"fields": {
"name": "Can change git ref",
- "content_type": 41,
+ "content_type": 40,
"codename": "change_gitref"
}
},
{
"model": "auth.permission",
- "pk": 160,
+ "pk": 156,
"fields": {
"name": "Can delete git ref",
- "content_type": 41,
+ "content_type": 40,
"codename": "delete_gitref"
}
},
{
"model": "auth.permission",
- "pk": 161,
+ "pk": 157,
"fields": {
"name": "Can view git ref",
- "content_type": 41,
+ "content_type": 40,
"codename": "view_gitref"
}
},
{
"model": "auth.permission",
- "pk": 162,
+ "pk": 158,
"fields": {
"name": "Can add process",
- "content_type": 42,
+ "content_type": 41,
"codename": "add_process"
}
},
{
"model": "auth.permission",
- "pk": 163,
+ "pk": 159,
"fields": {
"name": "Can change process",
- "content_type": 42,
+ "content_type": 41,
"codename": "change_process"
}
},
{
"model": "auth.permission",
- "pk": 164,
+ "pk": 160,
"fields": {
"name": "Can delete process",
- "content_type": 42,
+ "content_type": 41,
"codename": "delete_process"
}
},
{
"model": "auth.permission",
- "pk": 165,
+ "pk": 161,
"fields": {
"name": "Can view process",
- "content_type": 42,
+ "content_type": 41,
"codename": "view_process"
}
},
{
"model": "auth.permission",
- "pk": 166,
+ "pk": 162,
"fields": {
"name": "Can add process element",
- "content_type": 43,
+ "content_type": 42,
"codename": "add_processelement"
}
},
{
"model": "auth.permission",
- "pk": 167,
+ "pk": 163,
"fields": {
"name": "Can change process element",
- "content_type": 43,
+ "content_type": 42,
"codename": "change_processelement"
}
},
{
"model": "auth.permission",
- "pk": 168,
+ "pk": 164,
"fields": {
"name": "Can delete process element",
- "content_type": 43,
+ "content_type": 42,
"codename": "delete_processelement"
}
},
{
"model": "auth.permission",
- "pk": 169,
+ "pk": 165,
"fields": {
"name": "Can view process element",
- "content_type": 43,
+ "content_type": 42,
"codename": "view_processelement"
}
},
{
"model": "auth.permission",
- "pk": 170,
+ "pk": 166,
"fields": {
"name": "Can add repository",
- "content_type": 44,
+ "content_type": 43,
"codename": "add_repository"
}
},
{
"model": "auth.permission",
- "pk": 171,
+ "pk": 167,
"fields": {
"name": "Can change repository",
- "content_type": 44,
+ "content_type": 43,
"codename": "change_repository"
}
},
{
"model": "auth.permission",
- "pk": 172,
+ "pk": 168,
"fields": {
"name": "Can delete repository",
- "content_type": 44,
+ "content_type": 43,
"codename": "delete_repository"
}
},
{
"model": "auth.permission",
- "pk": 173,
+ "pk": 169,
"fields": {
"name": "Can view repository",
- "content_type": 44,
+ "content_type": 43,
"codename": "view_repository"
}
},
{
"model": "auth.permission",
- "pk": 174,
+ "pk": 170,
"fields": {
"name": "Can add revision",
- "content_type": 45,
+ "content_type": 44,
"codename": "add_revision"
}
},
{
"model": "auth.permission",
- "pk": 175,
+ "pk": 171,
"fields": {
"name": "Can change revision",
- "content_type": 45,
+ "content_type": 44,
"codename": "change_revision"
}
},
{
"model": "auth.permission",
- "pk": 176,
+ "pk": 172,
"fields": {
"name": "Can delete revision",
- "content_type": 45,
+ "content_type": 44,
"codename": "delete_revision"
}
},
{
"model": "auth.permission",
- "pk": 177,
+ "pk": 173,
"fields": {
"name": "Can view revision",
- "content_type": 45,
+ "content_type": 44,
"codename": "view_revision"
}
},
{
"model": "auth.permission",
- "pk": 178,
+ "pk": 174,
"fields": {
"name": "Can add worker",
- "content_type": 46,
+ "content_type": 45,
"codename": "add_worker"
}
},
{
"model": "auth.permission",
- "pk": 179,
+ "pk": 175,
"fields": {
"name": "Can change worker",
- "content_type": 46,
+ "content_type": 45,
"codename": "change_worker"
}
},
{
"model": "auth.permission",
- "pk": 180,
+ "pk": 176,
"fields": {
"name": "Can delete worker",
- "content_type": 46,
+ "content_type": 45,
"codename": "delete_worker"
}
},
{
"model": "auth.permission",
- "pk": 181,
+ "pk": 177,
"fields": {
"name": "Can view worker",
- "content_type": 46,
+ "content_type": 45,
"codename": "view_worker"
}
},
{
"model": "auth.permission",
- "pk": 182,
+ "pk": 178,
"fields": {
"name": "Can add worker activity",
- "content_type": 47,
+ "content_type": 46,
"codename": "add_workeractivity"
}
},
{
"model": "auth.permission",
- "pk": 183,
+ "pk": 179,
"fields": {
"name": "Can change worker activity",
- "content_type": 47,
+ "content_type": 46,
"codename": "change_workeractivity"
}
},
{
"model": "auth.permission",
- "pk": 184,
+ "pk": 180,
"fields": {
"name": "Can delete worker activity",
- "content_type": 47,
+ "content_type": 46,
"codename": "delete_workeractivity"
}
},
{
"model": "auth.permission",
- "pk": 185,
+ "pk": 181,
"fields": {
"name": "Can view worker activity",
- "content_type": 47,
+ "content_type": 46,
"codename": "view_workeractivity"
}
},
{
"model": "auth.permission",
- "pk": 186,
+ "pk": 182,
"fields": {
"name": "Can add worker configuration",
- "content_type": 48,
+ "content_type": 47,
"codename": "add_workerconfiguration"
}
},
{
"model": "auth.permission",
- "pk": 187,
+ "pk": 183,
"fields": {
"name": "Can change worker configuration",
- "content_type": 48,
+ "content_type": 47,
"codename": "change_workerconfiguration"
}
},
{
"model": "auth.permission",
- "pk": 188,
+ "pk": 184,
"fields": {
"name": "Can delete worker configuration",
- "content_type": 48,
+ "content_type": 47,
"codename": "delete_workerconfiguration"
}
},
{
"model": "auth.permission",
- "pk": 189,
+ "pk": 185,
"fields": {
"name": "Can view worker configuration",
- "content_type": 48,
+ "content_type": 47,
"codename": "view_workerconfiguration"
}
},
{
"model": "auth.permission",
- "pk": 190,
+ "pk": 186,
"fields": {
"name": "Can add worker type",
- "content_type": 49,
+ "content_type": 48,
"codename": "add_workertype"
}
},
{
"model": "auth.permission",
- "pk": 191,
+ "pk": 187,
"fields": {
"name": "Can change worker type",
- "content_type": 49,
+ "content_type": 48,
"codename": "change_workertype"
}
},
{
"model": "auth.permission",
- "pk": 192,
+ "pk": 188,
"fields": {
"name": "Can delete worker type",
- "content_type": 49,
+ "content_type": 48,
"codename": "delete_workertype"
}
},
{
"model": "auth.permission",
- "pk": 193,
+ "pk": 189,
"fields": {
"name": "Can view worker type",
- "content_type": 49,
+ "content_type": 48,
"codename": "view_workertype"
}
},
{
"model": "auth.permission",
- "pk": 194,
+ "pk": 190,
"fields": {
"name": "Can add worker version",
- "content_type": 50,
+ "content_type": 49,
"codename": "add_workerversion"
}
},
{
"model": "auth.permission",
- "pk": 195,
+ "pk": 191,
"fields": {
"name": "Can change worker version",
- "content_type": 50,
+ "content_type": 49,
"codename": "change_workerversion"
}
},
{
"model": "auth.permission",
- "pk": 196,
+ "pk": 192,
"fields": {
"name": "Can delete worker version",
- "content_type": 50,
+ "content_type": 49,
"codename": "delete_workerversion"
}
},
{
"model": "auth.permission",
- "pk": 197,
+ "pk": 193,
"fields": {
"name": "Can view worker version",
- "content_type": 50,
+ "content_type": 49,
"codename": "view_workerversion"
}
},
{
"model": "auth.permission",
- "pk": 198,
+ "pk": 194,
"fields": {
"name": "Can add worker run",
- "content_type": 51,
+ "content_type": 50,
"codename": "add_workerrun"
}
},
{
"model": "auth.permission",
- "pk": 199,
+ "pk": 195,
"fields": {
"name": "Can change worker run",
- "content_type": 51,
+ "content_type": 50,
"codename": "change_workerrun"
}
},
{
"model": "auth.permission",
- "pk": 200,
+ "pk": 196,
"fields": {
"name": "Can delete worker run",
- "content_type": 51,
+ "content_type": 50,
"codename": "delete_workerrun"
}
},
{
"model": "auth.permission",
- "pk": 201,
+ "pk": 197,
"fields": {
"name": "Can view worker run",
- "content_type": 51,
+ "content_type": 50,
"codename": "view_workerrun"
}
},
{
"model": "auth.permission",
- "pk": 202,
+ "pk": 198,
"fields": {
"name": "Can add process dataset",
- "content_type": 52,
+ "content_type": 51,
"codename": "add_processdataset"
}
},
{
"model": "auth.permission",
- "pk": 203,
+ "pk": 199,
"fields": {
"name": "Can change process dataset",
- "content_type": 52,
+ "content_type": 51,
"codename": "change_processdataset"
}
},
{
"model": "auth.permission",
- "pk": 204,
+ "pk": 200,
"fields": {
"name": "Can delete process dataset",
- "content_type": 52,
+ "content_type": 51,
"codename": "delete_processdataset"
}
},
{
"model": "auth.permission",
- "pk": 205,
+ "pk": 201,
"fields": {
"name": "Can view process dataset",
- "content_type": 52,
+ "content_type": 51,
"codename": "view_processdataset"
}
},
{
"model": "auth.permission",
- "pk": 206,
+ "pk": 202,
"fields": {
"name": "Can add dataset",
- "content_type": 53,
+ "content_type": 52,
"codename": "add_dataset"
}
},
{
"model": "auth.permission",
- "pk": 207,
+ "pk": 203,
"fields": {
"name": "Can change dataset",
- "content_type": 53,
+ "content_type": 52,
"codename": "change_dataset"
}
},
{
"model": "auth.permission",
- "pk": 208,
+ "pk": 204,
"fields": {
"name": "Can delete dataset",
- "content_type": 53,
+ "content_type": 52,
"codename": "delete_dataset"
}
},
{
"model": "auth.permission",
- "pk": 209,
+ "pk": 205,
"fields": {
"name": "Can view dataset",
- "content_type": 53,
+ "content_type": 52,
"codename": "view_dataset"
}
},
{
"model": "auth.permission",
- "pk": 210,
+ "pk": 206,
"fields": {
"name": "Can add metric key",
- "content_type": 54,
+ "content_type": 53,
"codename": "add_metrickey"
}
},
{
"model": "auth.permission",
- "pk": 211,
+ "pk": 207,
"fields": {
"name": "Can change metric key",
- "content_type": 54,
+ "content_type": 53,
"codename": "change_metrickey"
}
},
{
"model": "auth.permission",
- "pk": 212,
+ "pk": 208,
"fields": {
"name": "Can delete metric key",
- "content_type": 54,
+ "content_type": 53,
"codename": "delete_metrickey"
}
},
{
"model": "auth.permission",
- "pk": 213,
+ "pk": 209,
"fields": {
"name": "Can view metric key",
- "content_type": 54,
+ "content_type": 53,
"codename": "view_metrickey"
}
},
{
"model": "auth.permission",
- "pk": 214,
+ "pk": 210,
"fields": {
"name": "Can add model",
- "content_type": 55,
+ "content_type": 54,
"codename": "add_model"
}
},
{
"model": "auth.permission",
- "pk": 215,
+ "pk": 211,
"fields": {
"name": "Can change model",
- "content_type": 55,
+ "content_type": 54,
"codename": "change_model"
}
},
{
"model": "auth.permission",
- "pk": 216,
+ "pk": 212,
"fields": {
"name": "Can delete model",
- "content_type": 55,
+ "content_type": 54,
"codename": "delete_model"
}
},
{
"model": "auth.permission",
- "pk": 217,
+ "pk": 213,
"fields": {
"name": "Can view model",
- "content_type": 55,
+ "content_type": 54,
"codename": "view_model"
}
},
{
"model": "auth.permission",
- "pk": 218,
+ "pk": 214,
"fields": {
"name": "Can add model version",
- "content_type": 56,
+ "content_type": 55,
"codename": "add_modelversion"
}
},
{
"model": "auth.permission",
- "pk": 219,
+ "pk": 215,
"fields": {
"name": "Can change model version",
- "content_type": 56,
+ "content_type": 55,
"codename": "change_modelversion"
}
},
{
"model": "auth.permission",
- "pk": 220,
+ "pk": 216,
"fields": {
"name": "Can delete model version",
- "content_type": 56,
+ "content_type": 55,
"codename": "delete_modelversion"
}
},
{
"model": "auth.permission",
- "pk": 221,
+ "pk": 217,
"fields": {
"name": "Can view model version",
- "content_type": 56,
+ "content_type": 55,
"codename": "view_modelversion"
}
},
{
"model": "auth.permission",
- "pk": 222,
+ "pk": 218,
"fields": {
"name": "Can add metric value",
- "content_type": 57,
+ "content_type": 56,
"codename": "add_metricvalue"
}
},
{
"model": "auth.permission",
- "pk": 223,
+ "pk": 219,
"fields": {
"name": "Can change metric value",
- "content_type": 57,
+ "content_type": 56,
"codename": "change_metricvalue"
}
},
{
"model": "auth.permission",
- "pk": 224,
+ "pk": 220,
"fields": {
"name": "Can delete metric value",
- "content_type": 57,
+ "content_type": 56,
"codename": "delete_metricvalue"
}
},
{
"model": "auth.permission",
- "pk": 225,
+ "pk": 221,
"fields": {
"name": "Can view metric value",
- "content_type": 57,
+ "content_type": 56,
"codename": "view_metricvalue"
}
},
{
"model": "auth.permission",
- "pk": 226,
+ "pk": 222,
"fields": {
"name": "Can add dataset element",
- "content_type": 58,
+ "content_type": 57,
"codename": "add_datasetelement"
}
},
{
"model": "auth.permission",
- "pk": 227,
+ "pk": 223,
"fields": {
"name": "Can change dataset element",
- "content_type": 58,
+ "content_type": 57,
"codename": "change_datasetelement"
}
},
{
"model": "auth.permission",
- "pk": 228,
+ "pk": 224,
"fields": {
"name": "Can delete dataset element",
- "content_type": 58,
+ "content_type": 57,
"codename": "delete_datasetelement"
}
},
{
"model": "auth.permission",
- "pk": 229,
+ "pk": 225,
"fields": {
"name": "Can view dataset element",
- "content_type": 58,
+ "content_type": 57,
"codename": "view_datasetelement"
}
},
{
"model": "ponos.farm",
- "pk": "409b6859-63b9-41f2-8449-ba737bca6624",
+ "pk": "d4639943-3bc5-4839-ba11-bac03818050c",
"fields": {
"name": "Wheat farm",
- "seed": "9092a1fa53b98ee55ab02cf1d5e5bdb4652f2d020163b520e6c8e9cee0b0d3b4"
+ "seed": "0307fda00557ae52d5b623279314c220e0e132ac58897269042764e154194bea"
}
},
{
"model": "ponos.task",
- "pk": "ef80eff8-75a8-4fe6-8c7a-46300faaa342",
+ "pk": "67b4f068-55e2-4a60-b1be-5bba33971d4d",
"fields": {
"run": 0,
"depth": 0,
@@ -4020,21 +3979,22 @@
"agent": null,
"requires_gpu": false,
"gpu": null,
- "process": "9a08ea15-07be-4746-a0d9-7acca68e5c1b",
+ "process": "6f170a31-9cf3-4fef-a684-ce8c8a4000d2",
+ "worker_run": null,
"container": null,
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"expiry": "2100-12-31T23:59:59.999Z",
"extra_files": "{}",
- "token": "GoYSPBE0SraH8hiWN3QuGiDsG4Y+V0FVsNw3Q3h9J14=",
+ "token": "3lmDYOM7SpizY+afINTbz+34JrevUErTkEC21ubCc5w=",
"parents": []
}
},
{
"model": "ponos.artifact",
- "pk": "a93301c3-bdac-4d06-ba22-b7066decd282",
+ "pk": "27166636-75bd-4296-adce-ba8fa09a36c5",
"fields": {
- "task": "ef80eff8-75a8-4fe6-8c7a-46300faaa342",
+ "task": "67b4f068-55e2-4a60-b1be-5bba33971d4d",
"path": "/path/to/docker_build",
"size": 42000,
"content_type": "application/octet-stream",
@@ -4044,30 +4004,30 @@
},
{
"model": "training.dataset",
- "pk": "b7b3b1fa-b62a-4513-b99d-d4675e42525c",
+ "pk": "2233f882-da17-4468-94c0-f7ea5e8c8f69",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"creator": 2,
"task": null,
- "name": "Second Dataset",
- "description": "dataset number two",
+ "name": "First Dataset",
+ "description": "dataset number one",
"state": "open",
"sets": "[\"training\", \"test\", \"validation\"]"
}
},
{
"model": "training.dataset",
- "pk": "bf46bc61-d37c-49ba-9368-899de7524287",
+ "pk": "e8c15036-e0ea-4cc4-a370-981b739d5b55",
"fields": {
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
- "corpus": "76968a0c-d43f-4ba1-8cd0-0f7a0dcdaa68",
+ "corpus": "b925d835-ac8c-4f85-9f17-e2ba27cd4efa",
"creator": 2,
"task": null,
- "name": "First Dataset",
- "description": "dataset number one",
+ "name": "Second Dataset",
+ "description": "dataset number two",
"state": "open",
"sets": "[\"training\", \"test\", \"validation\"]"
}
diff --git a/arkindex/documents/management/commands/bootstrap.py b/arkindex/documents/management/commands/bootstrap.py
index 23dda2d5e2b8e70cd10a90720ff506411d4b290f..e6ba4f199a32b46f22001be21a6e219561672dff 100644
--- a/arkindex/documents/management/commands/bootstrap.py
+++ b/arkindex/documents/management/commands/bootstrap.py
@@ -158,9 +158,6 @@ class Command(BaseCommand):
# Create a fake worker version on a fake worker on a fake repo with a fake revision for file imports
repo, created = Repository.objects.get_or_create(
url=IMPORT_WORKER_REPO,
- defaults={
- "hook_token": str(uuid4()),
- }
)
if created:
self.success(f'Created Git repository for {IMPORT_WORKER_REPO}')
diff --git a/arkindex/documents/management/commands/build_fixtures.py b/arkindex/documents/management/commands/build_fixtures.py
index e7775bea98f0aeb1e8e326bd7dcf7fb9e7771aa7..cdc5f1282d4ceac416032797842c0241376d2d42 100644
--- a/arkindex/documents/management/commands/build_fixtures.py
+++ b/arkindex/documents/management/commands/build_fixtures.py
@@ -13,6 +13,7 @@ from arkindex.process.models import (
FeatureUsage,
Process,
ProcessMode,
+ Repository,
Worker,
WorkerRun,
WorkerType,
@@ -69,19 +70,9 @@ class Command(BaseCommand):
level=Role.Guest.value
)
- # Create OAuth credentials for a user
- creds = user.credentials.create(
- provider_url='https://somewhere',
- token='oauth-token',
- refresh_token='refresh-token',
- # Use an expiry very far away to avoid OAuth token refreshes in every test
- expiry=datetime(2100, 12, 31, 23, 59, 59, 999999, timezone.utc),
- )
-
# Create a GitLab worker repository
- gitlab_repo = creds.repos.create(
+ gitlab_repo = Repository.objects.create(
url='http://gitlab/repo',
- hook_token='hook-token',
)
# Create a revision on this repository
@@ -92,9 +83,8 @@ class Command(BaseCommand):
)
# Create another worker repository
- worker_repo = creds.repos.create(
+ worker_repo = Repository.objects.create(
url="http://my_repo.fake/workers/worker",
- hook_token='worker-hook-token',
)
# Create a revision on this repository
diff --git a/arkindex/documents/management/commands/load_export.py b/arkindex/documents/management/commands/load_export.py
index 97e9d23fd6352ccf660b2990b5406e87b47b7696..fe47c5343a4760146c2792d909fa6bd63625283c 100644
--- a/arkindex/documents/management/commands/load_export.py
+++ b/arkindex/documents/management/commands/load_export.py
@@ -2,7 +2,6 @@
import json
import os
import sqlite3
-import uuid
from datetime import datetime, timezone
from pathlib import Path
@@ -410,9 +409,6 @@ class Command(BaseCommand):
def create_repository(self, row):
repo, created = Repository.objects.get_or_create(
url=row['repository_url'],
- defaults={
- 'hook_token': str(uuid.uuid4()),
- },
)
return repo, created
diff --git a/arkindex/documents/tests/commands/test_load_export.py b/arkindex/documents/tests/commands/test_load_export.py
index 160399ae9b41b6dae3ab56cc7d1e7a011c2167e4..06512fc5e123b2d38d860ea6c2ca90ddc45995fd 100644
--- a/arkindex/documents/tests/commands/test_load_export.py
+++ b/arkindex/documents/tests/commands/test_load_export.py
@@ -33,7 +33,7 @@ class TestLoadExport(FixtureTestCase):
unexpected_fields_by_model = {
'documents.elementtype': ['display_name', 'indexable'],
'documents.mlclass': [],
- 'process.repository': ['hook_token', 'credentials', 'git_ref_revisions'],
+ 'process.repository': ['git_ref_revisions'],
'process.worker': [],
'process.revision': ['message', 'author'],
'process.workerversion': ['created', 'updated', 'configuration', 'state', 'docker_image', 'docker_image_iid'],
diff --git a/arkindex/documents/tests/tasks/test_corpus_delete.py b/arkindex/documents/tests/tasks/test_corpus_delete.py
index 3f112f1005fe3cfb3f8b42e39cb5322a894572a7..0b86d73b6c8913b5c8e467eb157267b87c0a42df 100644
--- a/arkindex/documents/tests/tasks/test_corpus_delete.py
+++ b/arkindex/documents/tests/tasks/test_corpus_delete.py
@@ -96,7 +96,6 @@ class TestDeleteCorpus(FixtureTestCase):
# Create a separate corpus that should not get anything deleted
cls.repo = Repository.objects.create(
url='http://lol.git',
- hook_token='h00k',
)
cls.corpus2 = Corpus.objects.create(name='Other corpus')
diff --git a/arkindex/process/admin.py b/arkindex/process/admin.py
index 22c24c2746671a60ae8583e72aa5ddd8e49a15f2..d3e44085cae805af3613a8b685fa7198ec7c4e01 100644
--- a/arkindex/process/admin.py
+++ b/arkindex/process/admin.py
@@ -76,7 +76,7 @@ class WorkerInline(admin.StackedInline):
class RepositoryAdmin(admin.ModelAdmin):
list_display = ('id', 'url')
- fields = ('id', 'url', 'hook_token', 'credentials')
+ fields = ('id', 'url')
readonly_fields = ('id', )
inlines = [WorkerInline, UserMembershipInline, GroupMembershipInline]
diff --git a/arkindex/process/api.py b/arkindex/process/api.py
index 3de0ba3b3a5cfcee32c1554ab89dbbadd4803a91..2f9f3ed381717fd885cddadef4afdc0cc212fae8 100644
--- a/arkindex/process/api.py
+++ b/arkindex/process/api.py
@@ -5,7 +5,6 @@ from textwrap import dedent
from uuid import UUID
from django.conf import settings
-from django.core.mail import send_mail
from django.db import transaction
from django.db.models import (
Avg,
@@ -24,7 +23,6 @@ from django.db.models import (
from django.db.models.functions import Coalesce, Now
from django.db.models.query import Prefetch
from django.shortcuts import get_object_or_404
-from django.template.loader import render_to_string
from django.utils.functional import cached_property
from drf_spectacular.utils import (
OpenApiExample,
@@ -51,7 +49,6 @@ from rest_framework.generics import (
)
from rest_framework.response import Response
from rest_framework.serializers import Serializer
-from rest_framework.views import APIView
from arkindex.documents.models import Corpus, Element, Selection
from arkindex.ponos.authentication import TaskAuthentication
@@ -65,7 +62,6 @@ from arkindex.process.models import (
Process,
ProcessDataset,
ProcessMode,
- Repository,
Revision,
Worker,
WorkerActivity,
@@ -75,9 +71,8 @@ from arkindex.process.models import (
WorkerType,
WorkerVersion,
)
-from arkindex.process.providers import GitProvider
from arkindex.process.serializers.files import DataFileCreateSerializer, DataFileSerializer
-from arkindex.process.serializers.git import ExternalRepositorySerializer, RevisionSerializer
+from arkindex.process.serializers.git import RevisionSerializer
from arkindex.process.serializers.imports import (
ApplyProcessTemplateSerializer,
CorpusProcessSerializer,
@@ -131,7 +126,7 @@ from arkindex.project.tools import PercentileCont
from arkindex.project.triggers import process_delete
from arkindex.training.models import Dataset, Model
from arkindex.training.serializers import DatasetSerializer
-from arkindex.users.models import OAuthCredentials, Role, Scope
+from arkindex.users.models import Role, Scope
logger = logging.getLogger(__name__)
@@ -448,8 +443,6 @@ class ProcessRetry(ProcessACLMixin, ProcessQuerysetMixin, GenericAPIView):
if process.mode == ProcessMode.Repository:
if not process.revision:
raise ValidationError({'__all__': ['Git repository imports must have a revision set']})
- if not process.revision.repo.enabled:
- raise ValidationError({'__all__': ['Git repository does not have any valid credentials']})
@extend_schema(
operation_id='RetryProcess',
@@ -773,45 +766,6 @@ class ProcessDatasetManage(CreateAPIView, DestroyAPIView):
return Response(status=status.HTTP_204_NO_CONTENT)
-@extend_schema(exclude=True)
-class GitRepositoryImportHook(APIView):
- """
- This endpoint is intended as a webhook for Git repository hosting applications like GitLab.
- """
-
- def post(self, request, pk=None, **kwargs):
- repo = get_object_or_404(Repository, id=pk)
- if not repo.enabled:
- raise PermissionDenied(detail='No credentials available for this repository.')
-
- try:
- repo.provider_class(credentials=repo.credentials).handle_webhook(repo, request)
- except Exception as e:
- user = repo.credentials.user
- # Notifying the user by mail that there was an error during webhook handling
- sent = send_mail(
- subject='An error occurred during Git hook handling',
- message=render_to_string(
- 'webhook_error.html',
- context={
- 'user': user,
- 'url': repo.url,
- 'error': e.args[0],
- },
- request=request,
- ),
- from_email=None,
- recipient_list=[user.email],
- fail_silently=True,
- )
- if sent == 0:
- logger.error(f'Failed to send webhook error email to {user.email}')
-
- raise
-
- return Response(status=status.HTTP_204_NO_CONTENT)
-
-
@extend_schema_view(
get=extend_schema(
tags=['repos'],
@@ -829,75 +783,10 @@ class RepositoryList(RepositoryACLMixin, ListAPIView):
def get_queryset(self):
return self.readable_repositories \
.annotate(authorized_users=Count('memberships')) \
- .select_related('credentials') \
.prefetch_related('workers__type') \
.order_by('url')
-@extend_schema_view(
- get=extend_schema(
- operation_id='ListExternalRepositories',
- parameters=[
- OpenApiParameter(
- 'search',
- description='Optional query terms to filter repositories',
- )
- ],
- tags=['repos'],
- ),
- post=extend_schema(
- operation_id='CreateExternalRepository',
- description='Using the given OAuth credentials, this links an external Git repository '
- 'to Arkindex, connects a push hook and starts an initial process.',
- responses={201: ProcessSerializer},
- tags=['repos'],
- )
-)
-class AvailableRepositoriesList(ListCreateAPIView):
- """
- List repositories associated to user OAuth credentials
-
- Using the given OAuth credentials ID, this uses the Git hosting
- application API's search feature to look for a repository matching
- the given query. Without a query, returns a full list.
- """
- permission_classes = (IsVerified, )
- pagination_class = None
- serializer_class = ExternalRepositorySerializer
- queryset = OAuthCredentials.objects.none()
-
- def get_queryset(self):
- cred = get_object_or_404(OAuthCredentials, user=self.request.user, id=self.kwargs['pk'])
- provider = cred.git_provider_class(credentials=cred)
- repos = provider.list_repos(query=self.request.query_params.get('search'))
- return map(provider.to_dict, repos)
-
- @transaction.atomic
- def create(self, request, *args, **kwargs):
- serializer = self.get_serializer(data=request.data)
- serializer.is_valid(raise_exception=True)
- user = self.request.user
- cred = get_object_or_404(OAuthCredentials, user=user, id=self.kwargs['pk'])
- provider = cred.git_provider_class(credentials=cred)
- repo = provider.create_repo(
- request=self.request,
- id=serializer.validated_data.get('id')
- )
- # Add an admin membership to the repository creator
- repo.memberships.create(user=user, level=Role.Admin.value)
-
- latest_commit_sha = provider.get_latest_commit_sha(repo)
- rev, _ = provider.get_or_create_revision(repo, latest_commit_sha)
-
- process = GitProvider.start_imports(provider, rev)
-
- # Return the serialized process
- return Response(
- status=status.HTTP_201_CREATED,
- data=ProcessSerializer(process, context={'request': self.request}).data,
- )
-
-
@extend_schema(
tags=['repos'],
description='Retrieve a repository',
diff --git a/arkindex/process/builder.py b/arkindex/process/builder.py
index 8469402e2de3766e1072980f8cdcc23428557bba..018703366c171302a1ba00b98e094faf93c76f40 100644
--- a/arkindex/process/builder.py
+++ b/arkindex/process/builder.py
@@ -187,8 +187,6 @@ class ProcessBuilder(object):
def validate_repository(self) -> None:
if self.process.revision is None:
raise ValidationError('A revision is required to create an import workflow from GitLab repository')
- if not self.process.revision.repo.enabled:
- raise ValidationError('Git repository does not have any valid credentials')
def validate_s3(self) -> None:
if not self.process.bucket_name:
diff --git a/arkindex/process/management/commands/update_repositories_hooks.py b/arkindex/process/management/commands/update_repositories_hooks.py
deleted file mode 100644
index cd3b07efbe0dcabdd22a1fb57284027a06c9c2e2..0000000000000000000000000000000000000000
--- a/arkindex/process/management/commands/update_repositories_hooks.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from django.core.management.base import BaseCommand
-
-from arkindex.process.models import Repository
-
-
-class Command(BaseCommand):
- help = "Update all Git repositories hooks"
-
- def add_arguments(self, parser):
- parser.add_argument(
- "base_url",
- type=str,
- help="Base url used to build the new hooks",
- )
-
- def handle(self, base_url, *args, **kwargs):
- for repository in Repository.objects.filter(credentials__isnull=False):
- self.stdout.write(f"Updating {repository}")
- repository.provider.create_hook(repository, base_url=base_url)
-
- self.stdout.write(self.style.SUCCESS("All done"))
diff --git a/arkindex/process/management/commands/update_repositories_refs.py b/arkindex/process/management/commands/update_repositories_refs.py
deleted file mode 100644
index 9f175c812eb6af9461fceed3589c0ba8f335eaa9..0000000000000000000000000000000000000000
--- a/arkindex/process/management/commands/update_repositories_refs.py
+++ /dev/null
@@ -1,32 +0,0 @@
-from django.core.management.base import BaseCommand
-
-from arkindex.process.models import Repository
-
-
-class Command(BaseCommand):
- help = "Update all Git repositories references and trigger build processes"
-
- def add_arguments(self, parser):
- parser.add_argument(
- "--repo-url",
- type=str,
- help="Limit process to one repository",
- )
-
- def handle(self, repo_url, *args, **kwargs):
-
- repositories = Repository.objects.all()
- if repo_url:
- repositories = repositories.filter(url=repo_url)
-
- for repository in repositories:
-
- self.stdout.write(f"Updating refs for {repository}")
-
- # List all refs
- try:
- repository.provider.update_repository_references(repository)
- except Exception as e:
- self.stderr.write(f"Failure for {repository}: {e}")
-
- self.stdout.write("All done")
diff --git a/arkindex/process/migrations/0026_remove_repository_unique_repository_hook_token_and_more.py b/arkindex/process/migrations/0026_remove_repository_unique_repository_hook_token_and_more.py
new file mode 100644
index 0000000000000000000000000000000000000000..78d38d5ff6a2e188d72ad300c02205ff9f8ae152
--- /dev/null
+++ b/arkindex/process/migrations/0026_remove_repository_unique_repository_hook_token_and_more.py
@@ -0,0 +1,29 @@
+# Generated by Django 4.1.7 on 2023-12-20 11:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('process', '0025_worker_archived'),
+ ]
+
+ operations = [
+ migrations.RunSQL(
+ "ALTER TABLE process_repository DROP CONSTRAINT unique_repository_hook_token",
+ elidable=True,
+ ),
+ migrations.RemoveConstraint(
+ model_name='repository',
+ name='unique_repository_hook_token',
+ ),
+ migrations.RemoveField(
+ model_name='repository',
+ name='credentials',
+ ),
+ migrations.RemoveField(
+ model_name='repository',
+ name='hook_token',
+ ),
+ ]
diff --git a/arkindex/process/models.py b/arkindex/process/models.py
index 28daa7f3abd5d15236abaa416ae6ff325db1767e..2280bda9a4e5e635ece0ff1518e0b17321394df0 100644
--- a/arkindex/process/models.py
+++ b/arkindex/process/models.py
@@ -23,7 +23,6 @@ from arkindex.process.managers import (
WorkerRunManager,
WorkerVersionManager,
)
-from arkindex.process.providers import get_provider
from arkindex.project.aws import S3FileMixin, S3FileStatus
from arkindex.project.fields import ArrayField, MD5HashField
from arkindex.project.models import IndexableModel
@@ -549,9 +548,6 @@ class DataFile(S3FileMixin, models.Model):
class Repository(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
url = models.URLField()
- hook_token = models.CharField(max_length=250)
- credentials = models.ForeignKey(
- 'users.OAuthCredentials', on_delete=models.SET_NULL, related_name='repos', blank=True, null=True)
git_ref_revisions = models.ManyToManyField('process.Revision', through='process.GitRef')
memberships = GenericRelation('users.Right', 'content_id')
@@ -561,32 +557,9 @@ class Repository(models.Model):
models.UniqueConstraint(
'url',
name='unique_repository_url',
- ),
- models.UniqueConstraint(
- 'hook_token',
- name='unique_repository_hook_token',
- ),
+ )
]
- # Make this constant into a database field to be able to handle repositories from services other than GitLab
- provider_name = 'GitLabProvider'
-
- @property
- def provider_class(self):
- if not self.provider_name:
- raise ValueError("Empty provider_name")
- return get_provider(self.provider_name)
-
- @property
- def provider(self):
- if self.provider_class is None:
- raise NotImplementedError(f"Missing provider {self.provider_name}")
- return self.provider_class(credentials=self.credentials)
-
- @property
- def enabled(self):
- return self.credentials is not None and self.provider_class is not None
-
def __str__(self):
return f'{self.url}'
diff --git a/arkindex/process/providers.py b/arkindex/process/providers.py
deleted file mode 100644
index 0d51f8539814c6fc5ff3400cd9b055a469873624..0000000000000000000000000000000000000000
--- a/arkindex/process/providers.py
+++ /dev/null
@@ -1,391 +0,0 @@
-import base64
-import logging
-import urllib.parse
-import uuid
-from abc import ABC, abstractmethod
-
-from django.conf import settings
-from django.urls import reverse
-from gitlab import Gitlab, GitlabCreateError, GitlabGetError
-from rest_framework.exceptions import APIException, AuthenticationFailed, NotAuthenticated, ValidationError
-
-from arkindex.process.utils import get_default_farm
-
-logger = logging.getLogger(__name__)
-
-
-class GitProvider(ABC):
-
- display_name = None
-
- def __init__(self, credentials=None):
- if credentials:
- from arkindex.users.models import OAuthCredentials
- assert isinstance(credentials, OAuthCredentials)
- self.credentials = credentials
-
- @abstractmethod
- def list_repos(self, query=None):
- """
- List all repositories or filter with a search query.
- """
-
- @abstractmethod
- def get_latest_commit_sha(self, repo):
- """
- Retrieve latest commit sha from the default branch on a given repository.
- """
-
- @abstractmethod
- def create_repo(self, **kwargs):
- """
- Create a Repository instance from an external repository
- """
-
- @abstractmethod
- def create_hook(self, repository, project_id=None, base_url=None):
- """Create a webhook to receive events from the project"""
-
- def update_or_create_ref(self, repo, revision, name, type):
- """
- Update or create a GitRef on a given repository:
- - If a GitRef with given name already exists on the given repository, we only update the
- linked revision with the given one
- - Else a GitRef is created on the given repository thanks to given name, type and revision
- """
- from arkindex.process.models import GitRef
- try:
- ref = repo.refs.get(name=name)
- ref.revision = revision
- ref.save()
- except GitRef.DoesNotExist:
- # Limit the name length to the max size of GitRef name field
- repo.refs.create(name=name[:250], type=type, revision=revision)
-
- def get_or_create_revision(self, repo, sha, save=True):
- from arkindex.process.models import Revision
- try:
- return self.get_revision(repo, sha), False
- except Revision.DoesNotExist:
- return self.create_revision(repo, sha, save=save), True
-
- def get_revision(self, repo, sha):
- return repo.revisions.get(hash=sha)
-
- @abstractmethod
- def create_revision(self, repo, sha, save=True):
- """
- Create a Revision instance for a given commit hash of a given repository.
- """
-
- @abstractmethod
- def handle_webhook(self, repo, request):
- """
- Handle a webhook event on a given repository.
- """
-
- @abstractmethod
- def to_dict(self, project):
- """
- Returns a dict specifying id, name and url from a repository
- """
-
- def start_imports(self, rev):
- """
- Create and start a process to build new worker(s) from a repository revision
- """
- from arkindex.process.models import ProcessMode
- mode = ProcessMode.Repository
- user = rev.repo.credentials.user
- refs = ", ".join(rev.refs.values_list('name', flat=True))
- if not refs:
- refs = rev.hash
- # Limit the name length to the max size of Process name field
- name = f"Import {refs} from {rev.repo.url}"[:100]
-
- farm = get_default_farm()
- if not farm.is_available(user):
- raise ValidationError("The owner of the OAuth credentials does not have access to the default farm.")
-
- process = rev.processes.create(
- creator=user,
- mode=mode,
- name=name,
- farm=farm,
- )
- process.run()
- return process
-
-
-class GitLabProvider(GitProvider):
-
- display_name = "GitLab"
-
- def _get_gitlab_client(self, credentials):
- if credentials.expired:
- credentials.provider.refresh_token()
-
- return Gitlab(credentials.provider_url, oauth_token=credentials.token)
-
- def _try_get_project(self, gl, id):
- try:
- return gl.projects.get(id)
- except GitlabGetError as e:
- raise APIException("Error while fetching GitLab project: {}".format(str(e)))
-
- def _get_project_from_repo(self, repo):
- assert repo.credentials, "Missing Gitlab credentials"
-
- gl = self._get_gitlab_client(repo.credentials)
- return self._try_get_project(gl, urllib.parse.urlsplit(repo.url).path.strip('/'))
-
- def list_repos(self, query=None):
- if not self.credentials:
- raise NotAuthenticated
- gl = self._get_gitlab_client(self.credentials)
- # Creating a webhook on a repo requires Maintainer (40) or Owner (50) access levels
- # See https://docs.gitlab.com/ce/api/members.html#valid-access-levels
- return gl.projects.list(min_access_level=40, search=query, get_all=False)
-
- def get_latest_commit_sha(self, repo):
- project = self._get_project_from_repo(repo)
- # Since ref_name isn't specified, we will use the repository default branch
- # See : https://docs.gitlab.com/ee/api/commits.html#list-repository-commits
- return project.commits.list(per_page=1, get_all=False)[0].id
-
- def to_dict(self, project):
- return {
- 'id': project.id,
- 'name': project.name_with_namespace,
- 'url': project.web_url
- }
-
- def create_repo(self, id=None, **kwargs):
- if not self.credentials:
- raise NotAuthenticated()
-
- gl = self._get_gitlab_client(self.credentials)
- project = self._try_get_project(gl, int(id))
-
- from arkindex.process.models import Repository
- if Repository.objects.filter(url=project.web_url).exists():
- raise ValidationError("A repository with this URL already exists")
-
- # Determine the user's access level on this project
- # When it is inherited from the group and not overridden in the project, project_access is not defined.
- # When the project isn't in a group, or the user is added to a specific project in a group,
- # group_access is not defined. project_access overrides group_access.
- access_level = 0
- if project.permissions.get('group_access'):
- access_level = project.permissions['group_access']['access_level']
- if project.permissions.get('project_access'):
- access_level = project.permissions['project_access']['access_level']
-
- # Maintainer level (40) is required
- if access_level < 40:
- raise ValidationError("Maintainer or Owner access is required to add a GitLab repository")
-
- repo = self.credentials.repos.create(
- url=project.web_url,
- hook_token=base64.b64encode(uuid.uuid4().bytes).decode('utf-8'),
- )
-
- self.create_hook(repo, project_id=int(id))
-
- return repo
-
- def create_hook(self, repository, project_id=None, base_url=None):
- """
- Configure the Gitlab hook to get events for a project.
-
- If `project_id` is set, then the project is retrieved directly from its GitLab project ID
- instead of matched using its path.
-
- The webhook's URL will use `base_url` as its base URL if it is set, and otherwise
- falls back on settings.BACKEND_PUBLIC_URL_OAUTH first, then settings.PUBLIC_HOSTNAME.
- """
- # Load project using a project ID or its repo path
- gitlab = Gitlab(repository.credentials.provider_url, oauth_token=repository.credentials.token)
- if project_id:
- project = self._try_get_project(gitlab, project_id)
- else:
- path = urllib.parse.urlparse(repository.url).path[1:]
- project = self._try_get_project(gitlab, path)
-
- if base_url is None:
- base_url = settings.BACKEND_PUBLIC_URL_OAUTH or settings.PUBLIC_HOSTNAME
-
- if base_url is None:
- raise APIException('Either the `base_url` argument, settings.BACKEND_PUBLIC_URL_OAUTH or settings.PUBLIC_HOSTNAME must be set.')
-
- url = urllib.parse.urljoin(base_url, reverse('api:import-hook', kwargs={'pk': repository.id}))
- logger.info(f"Webhook will be created as {url}")
-
- # Delete already configured hooks to be able to update
- for hook in project.hooks.list(all=True):
- if hook.url == url:
- hook.delete()
- logger.info(f"Deleted existing hook {hook.id}")
-
- try:
- # Create a new hook
- hook = project.hooks.create({
- 'url': url,
- 'push_events': True,
- "tag_push_events": True,
- 'token': repository.hook_token,
- })
- logger.info(f"Created new hook {hook.id}")
- except GitlabCreateError as e:
- raise APIException("Error while creating GitLab hook: {}".format(str(e)))
-
- def create_revision(self, repo, sha, save=True):
- from arkindex.process.models import GitRefType, Revision
-
- project = self._get_project_from_repo(repo)
- commit = project.commits.get(sha)
-
- rev = Revision(
- repo=repo,
- hash=sha,
- message=commit.message,
- author=commit.author_name,
- )
- if save:
- rev.save()
- for ref in commit.refs():
- try:
- ref_type = GitRefType(ref['type'])
- self.update_or_create_ref(repo, rev, ref['name'], ref_type)
- except ValueError:
- logger.warning(f'Git reference with type {ref["type"]} was ignored during revision creation')
- continue
-
- return rev
-
- def update_revision_references(self, repo, sha):
- """
- Update all references for a specific revision
- This is needed when a tag is pushed after its commit
- is already ingested
- """
- from arkindex.process.models import GitRefType
-
- project = self._get_project_from_repo(repo)
- commit = project.commits.get(sha)
-
- rev = repo.revisions.get(hash=sha)
- for ref in commit.refs():
- try:
- ref_type = GitRefType(ref['type'])
- self.update_or_create_ref(repo, rev, ref['name'], ref_type)
- except ValueError:
- logger.warning(f'Git reference with type {ref["type"]} was ignored during revision creation')
- continue
-
- def update_repository_references(self, repo):
- """
- List all available references (branches and tags) on a remote repository
- and create new references or update existing ones
- New import process are started for new references
- """
- from arkindex.process.models import GitRefType
- project = self._get_project_from_repo(repo)
-
- def _update_reference(ref, ref_type):
- assert isinstance(ref_type, GitRefType)
- try:
- rev, created = repo.revisions.get_or_create(
- hash=ref.commit['id'],
- defaults={
- 'message': ref.commit['message'],
- 'author': ref.commit['author_email'],
- }
- )
- self.update_or_create_ref(repo, rev, ref.name, ref_type)
- if created:
- logger.info(f"Starting import for {ref_type.value} {ref.name}")
- self.start_imports(rev)
- except ValueError:
- logger.warning(f'{ref_type.value} {ref.name} was ignored during revision creation')
-
- for branch in project.branches.list(get_all=True):
- _update_reference(branch, GitRefType.Branch)
-
- for tag in project.tags.list(get_all=True):
- _update_reference(tag, GitRefType.Tag)
-
- def handle_webhook(self, repo, request):
- from arkindex.process.models import GitRefType
-
- if 'HTTP_X_GITLAB_EVENT' not in request.META:
- raise ValidationError("Missing GitLab event type")
-
- if request.META['HTTP_X_GITLAB_EVENT'] not in ('Push Hook', 'Tag Push Hook'):
- raise ValidationError("Unsupported GitLab event type")
-
- if 'HTTP_X_GITLAB_TOKEN' not in request.META:
- raise NotAuthenticated("Missing GitLab secret token")
- if request.META['HTTP_X_GITLAB_TOKEN'] != repo.hook_token:
- raise AuthenticationFailed("Invalid GitLab secret token")
-
- if not isinstance(request.data, dict) or 'object_kind' not in request.data:
- raise ValidationError('Bad payload format')
-
- kind = request.data.get('object_kind')
- if kind not in ('push', 'tag_push'):
- raise ValidationError("Unsupported GitLab event type")
-
- sha = request.data.get('checkout_sha')
- if kind == 'tag_push':
- if sha:
- # When a commit SHA is present, we need to add a reference on an existing commit
- self.update_revision_references(repo, sha)
-
- else:
- # If there isn't any SHA it means that a tag was deleted
- ref = request.data.get('ref')
- if not ref:
- raise ValidationError('Missing tag reference')
-
- # Delete existing tag
- tag_name = ref[10:] if ref.startswith('refs/tags/') else ref
- repo.refs.filter(name=tag_name, type=GitRefType.Tag).delete()
-
- return
-
- if not sha:
- # If there isn't any SHA it means that a branch was deleted
- ref = request.data.get('ref')
- if not ref:
- raise ValidationError('Missing branch reference')
-
- # Delete existing branch
- branch_name = ref[11:] if ref.startswith('refs/heads/') else ref
- repo.refs.filter(name=branch_name, type=GitRefType.Branch).delete()
-
- return
-
- # Already took care of this event
- if repo.revisions.filter(hash=sha).exists():
- return
-
- rev = self.create_revision(repo, sha)
-
- self.start_imports(rev)
-
-
-git_providers = [
- GitLabProvider,
-]
-oauth_to_git = {
- "gitlab": GitLabProvider,
-}
-
-
-def get_provider(name):
- return next(filter(lambda p: p.__name__ == name, git_providers), None)
-
-
-def from_oauth(name):
- return oauth_to_git.get(name)
diff --git a/arkindex/process/serializers/git.py b/arkindex/process/serializers/git.py
index 30dde44255665a2403410f992af211991267c001..eb2027c6cbc5695566f75b2375e9bc6be6b823a6 100644
--- a/arkindex/process/serializers/git.py
+++ b/arkindex/process/serializers/git.py
@@ -58,12 +58,3 @@ class RevisionWithRefsSerializer(serializers.ModelSerializer):
'commit_url',
'refs',
)
-
-
-class ExternalRepositorySerializer(serializers.Serializer):
- """
- Serialize a Git repository from an external API
- """
- id = serializers.IntegerField(min_value=0)
- name = serializers.CharField(read_only=True)
- url = serializers.URLField(read_only=True)
diff --git a/arkindex/process/serializers/workers.py b/arkindex/process/serializers/workers.py
index 7c91b90adc3f3cd08a3abc2781279e887e44e429..47e994e1c18cdf0e4e4eba85121e7ba267aa658a 100644
--- a/arkindex/process/serializers/workers.py
+++ b/arkindex/process/serializers/workers.py
@@ -1,6 +1,3 @@
-import base64
-import urllib
-import uuid
from collections import defaultdict
from enum import Enum
from textwrap import dedent
@@ -8,7 +5,6 @@ from textwrap import dedent
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.db.models import Max, Q
-from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError
@@ -390,29 +386,9 @@ class RepositorySerializer(serializers.ModelSerializer):
"""
Serialize a repository
"""
- enabled = serializers.BooleanField(read_only=True)
- git_clone_url = serializers.SerializerMethodField()
workers = WorkerLightSerializer(many=True, read_only=True)
authorized_users = serializers.SerializerMethodField(read_only=True)
- @extend_schema_field(serializers.CharField(allow_null=True))
- def get_git_clone_url(self, repository):
- # This check avoid to set git_clone_url when this serializer is used to list multiple
- # repositories because self.instance value would be a list, even with Ponos task authentication.
- # We also restrict to enabled repositories as disabled repos do not have OAuth credentials,
- # and restrict to the repository set on the task's process, as it should only clone that one.
- process = get_process_from_task_auth(self.context['request'])
- if (
- process
- and isinstance(self.instance, Repository)
- and self.instance.enabled
- and process.revision_id is not None
- and process.revision.repo_id == self.instance.id
- ):
- url = urllib.parse.urlparse(self.instance.url)
- return f"https://oauth2:{repository.credentials.token}@{url.netloc}{url.path}"
- return None
-
def get_authorized_users(self, repo) -> int:
count = getattr(repo, 'authorized_users', None)
if count is None:
@@ -424,8 +400,6 @@ class RepositorySerializer(serializers.ModelSerializer):
fields = (
'id',
'url',
- 'enabled',
- 'git_clone_url',
'workers',
'authorized_users',
)
@@ -686,11 +660,7 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
"""
# Retrieve or create the Git repository
repository, created_repo = Repository.objects.using('default').get_or_create(
- url=validated_data['repository_url'],
- defaults={
- # Generate a default hook token (DB constraint) even if no webhook is created
- 'hook_token': base64.b64encode(uuid.uuid4().bytes).decode('utf-8')
- },
+ url=validated_data['repository_url']
)
# Grant an admin access to the repository in case it got created
if created_repo:
diff --git a/arkindex/process/tests/test_create_process.py b/arkindex/process/tests/test_create_process.py
index d3fc48d8f8c14fda33c1206ca1ee046fa2901f38..92de6c29d908c184922ad98e6b7030c808146402 100644
--- a/arkindex/process/tests/test_create_process.py
+++ b/arkindex/process/tests/test_create_process.py
@@ -12,6 +12,7 @@ from arkindex.process.models import (
ActivityState,
Process,
ProcessMode,
+ Repository,
WorkerActivity,
WorkerVersion,
WorkerVersionState,
@@ -41,8 +42,7 @@ class TestCreateProcess(FixtureAPITestCase):
cls.private_ml_class = cls.private_corpus.ml_classes.create(name='chouquette')
# Workers ProcessMode
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
cls.rev_1 = cls.repo.revisions.get()
cls.rev_2 = cls.repo.revisions.create(
hash='2',
diff --git a/arkindex/process/tests/test_docker_worker_version.py b/arkindex/process/tests/test_docker_worker_version.py
index 2b7f92a0557d6f98319c9b50c89f127675bc5d71..2576b7b8d1bca0481aeeb954f267c33fd24eb3ae 100644
--- a/arkindex/process/tests/test_docker_worker_version.py
+++ b/arkindex/process/tests/test_docker_worker_version.py
@@ -17,8 +17,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
@classmethod
def setUpTestData(cls):
super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
cls.rev = cls.repo.revisions.get()
cls.worker = Worker.objects.get(slug='reco')
cls.version = cls.worker.versions.get()
diff --git a/arkindex/process/tests/test_gitlab_provider.py b/arkindex/process/tests/test_gitlab_provider.py
deleted file mode 100644
index b7bc455074ecfaaa5c1d3974f2926def9c6a5919..0000000000000000000000000000000000000000
--- a/arkindex/process/tests/test_gitlab_provider.py
+++ /dev/null
@@ -1,786 +0,0 @@
-import collections
-from pathlib import Path
-from unittest.mock import MagicMock, patch
-
-import responses
-from django.test import override_settings
-from gitlab.exceptions import GitlabCreateError, GitlabGetError
-from responses import matchers
-from rest_framework.exceptions import APIException, AuthenticationFailed, NotAuthenticated, ValidationError
-
-from arkindex.process.models import GitRefType, Process, ProcessMode, Revision
-from arkindex.process.providers import GitLabProvider
-from arkindex.project.tests import FixtureTestCase
-
-SAMPLES = Path(__file__).absolute().parent / 'repository_conf_samples'
-
-
-class TestGitLabProvider(FixtureTestCase):
- """
- Test the GitLabProvider class
- """
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://gitlab/repo')
- cls.rev = cls.repo.revisions.get()
- cls.gl_patch = patch('arkindex.process.providers.Gitlab')
-
- def setUp(self):
- super().setUp()
- self.gl_mock = self.gl_patch.start()
-
- def tearDown(self):
- super().tearDown()
- self.gl_patch.stop()
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_list_repos(self):
- """
- Test GitLabProvider can list repositories from GitLab
- """
- GitLabProvider(credentials=self.creds).list_repos()
-
- self.assertEqual(self.gl_mock.call_count, 1)
- args, kwargs = self.gl_mock.call_args
- self.assertTupleEqual(args, ('https://somewhere', ))
- self.assertDictEqual(kwargs, {'oauth_token': self.creds.token})
-
- self.assertEqual(self.gl_mock().projects.list.call_count, 1)
- args, kwargs = self.gl_mock().projects.list.call_args
- self.assertTupleEqual(args, ())
- self.assertDictEqual(kwargs, {'get_all': False, 'min_access_level': 40, 'search': None})
-
- @responses.activate
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost', GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t')
- def test_list_repos_refresh(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'refresh-token'
- }),
- ],
- json={
- 'access_token': 'new-token',
- 'refresh_token': 'new-refresh-token',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
- responses.get('https://somewhere/api/v4/user', json={'id': 42, 'username': 'Someone'})
- self.creds.expiry = None
- self.creds.save()
-
- GitLabProvider(credentials=self.creds).list_repos()
-
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.token, 'new-token')
- self.assertEqual(self.creds.refresh_token, 'new-refresh-token')
- self.assertEqual(self.creds.account_name, 'Someone')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_list_repos_query(self):
- """
- Test GitLabProvider can search repositories from GitLab
- """
- GitLabProvider(credentials=self.creds).list_repos(query='meh')
-
- self.assertEqual(self.gl_mock.call_count, 1)
- args, kwargs = self.gl_mock.call_args
- self.assertTupleEqual(args, ('https://somewhere', ))
- self.assertDictEqual(kwargs, {'oauth_token': self.creds.token})
-
- self.assertEqual(self.gl_mock().projects.list.call_count, 1)
- args, kwargs = self.gl_mock().projects.list.call_args
- self.assertTupleEqual(args, ())
- self.assertDictEqual(kwargs, {'get_all': False, 'min_access_level': 40, 'search': 'meh'})
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_list_repos_requires_credentials(self):
- """
- Test GitLabProvider checks for credentials when requesting repositories list
- """
- with self.assertRaises(NotAuthenticated):
- GitLabProvider().list_repos()
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_repo(self):
- """
- Test GitLabProvider can create a Repository instance from a GitLab repo
- """
- self.gl_mock().projects.get.return_value.web_url = 'http://new_repo_url'
- self.gl_mock().projects.get.return_value.permissions = {
- 'project_access': {'access_level': 50},
- 'group_access': None
- }
-
- glp = GitLabProvider(credentials=self.creds)
-
- new_repo = glp.create_repo(id='1337')
-
- self.assertEqual(self.gl_mock().projects.get.call_count, 2)
- args, kwargs = self.gl_mock().projects.get.call_args
- self.assertTupleEqual(args, (1337, ))
- self.assertDictEqual(kwargs, {})
-
- self.assertEqual(new_repo.url, 'http://new_repo_url')
- self.assertEqual(new_repo.provider_name, 'GitLabProvider')
-
- self.assertEqual(self.gl_mock().projects.get().hooks.create.call_count, 1)
- args, kwargs = self.gl_mock().projects.get().hooks.create.call_args
- self.assertEqual(len(args), 1)
- self.assertDictEqual(kwargs, {})
- self.assertDictEqual(args[0], {
- 'url': f'https://arkindex.localhost/api/v1/imports/hook/{new_repo.id}/',
- 'push_events': True,
- 'tag_push_events': True,
- 'token': new_repo.hook_token,
- })
-
- @responses.activate
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost', GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t')
- def test_create_repo_refresh(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'refresh-token'
- }),
- ],
- json={
- 'access_token': 'new-token',
- 'refresh_token': 'new-refresh-token',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
- responses.get('https://somewhere/api/v4/user', json={'id': 42, 'username': 'Someone'})
- self.creds.expiry = None
- self.creds.save()
-
- self.gl_mock().projects.get.return_value.web_url = 'http://new_repo_url'
- self.gl_mock().projects.get.return_value.permissions = {
- 'project_access': {'access_level': 50},
- 'group_access': None
- }
-
- glp = GitLabProvider(credentials=self.creds)
- glp.create_repo(id='1337')
-
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.token, 'new-token')
- self.assertEqual(self.creds.refresh_token, 'new-refresh-token')
- self.assertEqual(self.creds.account_name, 'Someone')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_repo_requires_credentials(self):
- """
- Test GitLabProvider checks for credentials when requesting a repository creation
- """
- with self.assertRaises(NotAuthenticated):
- GitLabProvider().create_repo(id='repo_id')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_repo_already_exists(self):
- """
- Test GitLabProvider checks for duplicate repositories
- """
- self.gl_mock().projects.get.return_value.web_url = 'http://new_repo_url'
- self.gl_mock().projects.get.return_value.permissions = {
- 'project_access': {'access_level': 40},
- 'group_access': None
- }
-
- glp = GitLabProvider(credentials=self.creds)
- glp.create_repo(id='1337')
-
- with self.assertRaises(ValidationError):
- GitLabProvider(credentials=self.creds).create_repo(
- id='1337')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_repo_requires_maintainer(self):
- """
- Test GitLabProvider checks for duplicate repositories
- """
- self.gl_mock().projects.get.return_value.web_url = 'http://new_repo_url'
- self.gl_mock().projects.get.return_value.permissions = {
- 'group_access': {'access_level': 30},
- 'project_access': None,
- }
-
- glp = GitLabProvider(credentials=self.creds)
-
- with self.assertRaisesRegex(
- ValidationError,
- 'Maintainer or Owner access is required to add a GitLab repository'):
- glp.create_repo(id='1337')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_repo_handle_get_error(self):
- """
- Test GitLabProvider handles GitLab repo GET errors
- """
- self.gl_mock().projects.get.side_effect = GitlabGetError
-
- with self.assertRaises(APIException):
- GitLabProvider(credentials=self.creds).create_repo(id='1337')
-
- self.assertEqual(self.gl_mock().projects.get.call_count, 1)
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_repo_handle_hook_create_error(self):
- """
- Test GitLabProvider handles GitLab hook creation errors
- """
- self.gl_mock().projects.get.return_value.web_url = 'http://new_repo_url'
- self.gl_mock().projects.get.return_value.permissions = {
- 'project_access': {'access_level': 50}
- }
- self.gl_mock().projects.get.return_value.hooks.create.side_effect = GitlabCreateError
-
- with self.assertRaises(APIException):
- glp = GitLabProvider(credentials=self.creds)
- glp.create_repo(id='1337')
-
- self.assertEqual(self.gl_mock().projects.get.call_count, 2)
- self.assertEqual(self.gl_mock().projects.get().hooks.create.call_count, 1)
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_update_or_create_ref(self):
- """
- Test GitLabProvider can create or update a GitRef instance for a repo and a revision
- """
- commit_refs = [
- {'name': 'refs/tags/v0.1.0', 'type': 'tag'},
- {'name': 'refs/heads/branch1', 'type': 'branch'},
- {'name': 'refs/heads/branch2', 'type': 'branch'},
- ]
- rev1 = Revision(
- repo=self.repo,
- hash='1',
- message='commit message',
- author='bob',
- )
- rev1.save()
- rev2 = Revision(
- repo=self.repo,
- hash='2',
- message='commit message',
- author='bob',
- )
- rev2.save()
-
- # Assert that git references are created properly
- for ref in commit_refs:
- GitLabProvider(credentials=self.creds) \
- .update_or_create_ref(self.repo, rev1, ref['name'], ref['type'])
-
- refs = [
- {'name': ref.name, 'type': ref.type, 'repo': ref.repository}
- for ref in rev1.refs.all()
- ]
- self.assertEqual(len(refs), 3)
- self.assertListEqual(refs, [
- {'name': 'refs/tags/v0.1.0', 'type': GitRefType.Tag, 'repo': self.repo},
- {'name': 'refs/heads/branch1', 'type': GitRefType.Branch, 'repo': self.repo},
- {'name': 'refs/heads/branch2', 'type': GitRefType.Branch, 'repo': self.repo},
- ])
- self.assertEqual(len(self.repo.refs.all()), 3)
-
- # Assert that git references are updated with another revision properly
- GitLabProvider(credentials=self.creds) \
- .update_or_create_ref(self.repo, rev2, commit_refs[0]['name'], commit_refs[0]['type'])
- GitLabProvider(credentials=self.creds) \
- .update_or_create_ref(self.repo, rev2, commit_refs[1]['name'], commit_refs[1]['type'])
-
- refs_rev1 = [
- {'name': ref.name, 'type': ref.type, 'repo': ref.repository}
- for ref in rev1.refs.all()
- ]
- refs_rev2 = [
- {'name': ref.name, 'type': ref.type, 'repo': ref.repository}
- for ref in rev2.refs.all()
- ]
- self.assertEqual(len(refs_rev1), 1)
- self.assertListEqual(refs_rev1, [
- {'name': 'refs/heads/branch2', 'type': GitRefType.Branch, 'repo': self.repo},
- ])
- self.assertEqual(len(refs_rev2), 2)
- self.assertListEqual(refs_rev2, [
- {'name': 'refs/tags/v0.1.0', 'type': GitRefType.Tag, 'repo': self.repo},
- {'name': 'refs/heads/branch1', 'type': GitRefType.Branch, 'repo': self.repo},
- ])
- self.assertEqual(len(self.repo.refs.all()), 3)
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_get_revision(self):
- """
- Test GitLabProvider can create a Revision instance for a repo by hash
- """
- revision, created = GitLabProvider(credentials=self.creds) \
- .get_or_create_revision(self.repo, '42')
-
- self.assertEqual(revision, self.rev)
- self.assertFalse(created)
- self.assertEqual(self.gl_mock.call_count, 0)
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_create_revision(self):
- """
- Test GitLabProvider can create a Revision instance for a repo by hash
- """
- self.gl_mock().projects.get.return_value.commits.get.return_value.refs.return_value = [
- {'name': 'refs/tags/v0.1.0', 'type': 'tag'},
- {'name': 'refs/heads/branch1', 'type': 'branch'},
- {'name': 'refs/heads/branch2', 'type': 'branch'},
- {'name': 'oh_no', 'type': 'all'},
- ]
- self.gl_mock().projects.get.return_value.commits.get.return_value.message = 'commit message'
- self.gl_mock().projects.get.return_value.commits.get.return_value.author_name = 'bob'
-
- revision, created = GitLabProvider(credentials=self.creds) \
- .get_or_create_revision(self.repo, '1337')
-
- self.assertTrue(created)
- self.assertEqual(revision.hash, '1337')
-
- # Assert that git references are created properly
- refs = [
- {
- 'name': ref.name,
- 'type': ref.type,
- 'repo': ref.repository,
- }
- for ref in revision.refs.all()
- ]
- self.assertEqual(len(refs), 3)
- self.assertListEqual(refs, [
- {'name': 'refs/tags/v0.1.0', 'type': GitRefType.Tag, 'repo': self.repo},
- {'name': 'refs/heads/branch1', 'type': GitRefType.Branch, 'repo': self.repo},
- {'name': 'refs/heads/branch2', 'type': GitRefType.Branch, 'repo': self.repo},
- ])
-
- self.assertEqual(revision.message, 'commit message')
- self.assertEqual(revision.author, 'bob')
-
- self.assertEqual(self.gl_mock().projects.get.call_count, 1)
- self.assertEqual(self.gl_mock().projects.get().commits.get.call_count, 1)
- args, kwargs = self.gl_mock().projects.get().commits.get.call_args
- self.assertTupleEqual(args, ('1337', ))
- self.assertDictEqual(kwargs, {})
-
- @responses.activate
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost', GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t')
- def test_create_revision_refresh(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'refresh-token'
- }),
- ],
- json={
- 'access_token': 'new-token',
- 'refresh_token': 'new-refresh-token',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
- responses.get('https://somewhere/api/v4/user', json={'id': 42, 'username': 'Someone'})
- self.creds.expiry = None
- self.creds.save()
-
- self.gl_mock().projects.get.return_value.commits.get.return_value.refs.return_value = []
- self.gl_mock().projects.get.return_value.commits.get.return_value.message = 'commit message'
- self.gl_mock().projects.get.return_value.commits.get.return_value.author_name = 'bob'
-
- GitLabProvider(credentials=self.creds).get_or_create_revision(self.repo, '1337')
-
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.token, 'new-token')
- self.assertEqual(self.creds.refresh_token, 'new-refresh-token')
- self.assertEqual(self.creds.account_name, 'Someone')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_handle_webhook_missing_headers(self):
- """
- Test GitLabProvider checks HTTP headers on webhooks
- """
- glp = GitLabProvider(credentials=self.creds)
-
- request_mock = MagicMock()
- request_mock.data = {
- 'object_kind': 'push',
- 'ref': 'refs/heads/master',
- 'checkout_sha': '1337',
- 'commits': [
- {
- 'message': 'commit message',
- 'author': {
- 'name': 'bob',
- }
- }
- ]
- }
-
- # Missing HTTP_X_GITLAB_TOKEN
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- }
- with self.assertRaises(NotAuthenticated):
- glp.handle_webhook(self.repo, request_mock)
-
- # Missing HTTP_X_GITLAB_EVENT
- request_mock.META = {
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- with self.assertRaises(ValidationError):
- glp.handle_webhook(self.repo, request_mock)
-
- # Wrong HTTP_X_GITLAB_EVENT
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Not a Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- with self.assertRaises(ValidationError):
- glp.handle_webhook(self.repo, request_mock)
-
- # Wrong HTTP_X_GITLAB_TOKEN
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'not-the-hook-token',
- }
- with self.assertRaises(AuthenticationFailed):
- glp.handle_webhook(self.repo, request_mock)
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_handle_webhook_create_revision(self):
- """
- Test GitLabProvider does create a revision when a push event is received
- """
- request_mock = MagicMock()
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- sha = '1337'
- request_mock.data = {
- 'object_kind': 'push',
- 'ref': 'refs/heads/master',
- 'checkout_sha': sha,
- 'commits': [
- {
- 'message': 'commit message',
- 'author': {
- 'name': 'bob',
- }
- }
- ]
- }
-
- self.gl_mock().projects.get.return_value.commits.get.return_value.refs.return_value = [
- {'name': 'oh_no', 'type': 'tag'},
- {'name': 'refs/heads/branch1', 'type': 'branch'},
- {'name': 'refs/heads/branch2', 'type': 'branch'},
- ]
- self.gl_mock().projects.get.return_value.commits.get.return_value.message = 'commit message'
- self.gl_mock().projects.get.return_value.commits.get.return_value.author_name = 'bob'
-
- rev = self.repo.revisions.filter(hash=sha)
- self.assertFalse(rev.exists())
- repo_processes = Process.objects.filter(revision__repo_id=str(self.repo.id))
- self.assertFalse(repo_processes.exists())
- GitLabProvider(credentials=self.creds).handle_webhook(self.repo, request_mock)
-
- di = repo_processes.get()
- self.assertEqual(di.mode, ProcessMode.Repository)
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_handle_webhook_duplicate_events(self):
- """
- Test GitLabProvider checks for already handled events
- """
- request_mock = MagicMock()
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- sha = '42'
- request_mock.data = {
- 'object_kind': 'push',
- 'ref': 'refs/heads/master',
- 'checkout_sha': sha,
- 'commits': [
- {
- 'message': 'a',
- 'author': {
- 'name': 'me',
- }
- }
- ]
- }
-
- rev = self.repo.revisions.filter(hash=sha)
- self.assertTrue(rev.exists())
- repo_processes = Process.objects.filter(revision__repo_id=str(self.repo.id))
- self.assertFalse(repo_processes.exists())
- GitLabProvider(credentials=self.creds).handle_webhook(self.repo, request_mock)
-
- # Checking that we didn't initiate revision creation
- self.assertEqual(self.gl_mock().projects.get.call_count, 0)
- self.assertEqual(self.gl_mock().projects.get().commits.get.call_count, 0)
- self.assertFalse(repo_processes.exists())
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_handle_webhook_wrong_kind(self):
- """
- Test GitLabProvider gracefully fails when GitLab breaks things
- """
- sha = '1337'
- rev = self.repo.revisions.filter(hash=sha)
- self.assertFalse(rev.exists())
- repo_processes = Process.objects.filter(revision__repo_id=str(self.repo.id))
-
- glp = GitLabProvider(credentials=self.creds)
- request_mock = MagicMock()
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- request_mock.data = {
- 'object_kind': 'oh_no', # Bad object kind
- 'ref': 'refs/heads/master',
- 'checkout_sha': sha,
- 'commits': [
- {
- 'message': 'b',
- 'author': {
- 'name': 'me',
- }
- }
- ]
- }
- with self.assertRaises(ValidationError):
- glp.handle_webhook(self.repo, request_mock)
- self.assertFalse(rev.exists())
- self.assertFalse(repo_processes.exists())
-
- # Breaking change: a list!
- request_mock.data = [request_mock.data]
- with self.assertRaises(ValidationError):
- glp.handle_webhook(self.repo, request_mock)
- self.assertFalse(rev.exists())
- self.assertFalse(repo_processes.exists())
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_handle_webhook_delete_branch(self):
- """
- Test GitLabProvider properly handles a branch deletion
- """
- rev = Revision(
- repo=self.repo,
- hash='1',
- message='commit message',
- author='bob',
- )
- rev.save()
- self.assertTrue(self.repo.revisions.filter(hash='1').exists())
- repo_processes = Process.objects.filter(revision__repo_id=str(self.repo.id))
-
- glp = GitLabProvider(credentials=self.creds)
- glp.update_or_create_ref(self.repo, rev, 'test', GitRefType.Branch)
- self.assertEqual(len(self.repo.refs.all()), 1)
- request_mock = MagicMock()
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- request_mock.data = {
- 'object_kind': 'push',
- 'ref': 'refs/heads/test',
- 'commits': []
- }
-
- glp.handle_webhook(self.repo, request_mock)
- self.assertTrue(self.repo.revisions.filter(hash='1').exists())
- self.assertEqual(len(self.repo.refs.all()), 0)
- self.assertFalse(repo_processes.exists())
-
- @responses.activate
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost', GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t')
- def test_handle_webhook_refresh(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'refresh-token'
- }),
- ],
- json={
- 'access_token': 'new-token',
- 'refresh_token': 'new-refresh-token',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
- responses.get('https://somewhere/api/v4/user', json={'id': 42, 'username': 'Someone'})
- self.creds.expiry = None
- self.creds.save()
-
- self.gl_mock().projects.get.return_value.commits.get.return_value.refs.return_value = []
- self.gl_mock().projects.get.return_value.commits.get.return_value.message = 'commit message'
- self.gl_mock().projects.get.return_value.commits.get.return_value.author_name = 'bob'
-
- request_mock = MagicMock()
- request_mock.META = {
- 'HTTP_X_GITLAB_EVENT': 'Push Hook',
- 'HTTP_X_GITLAB_TOKEN': 'hook-token',
- }
- request_mock.data = {
- 'object_kind': 'push',
- 'ref': 'refs/heads/something',
- 'commits': [],
- 'checkout_sha': '1337',
- }
-
- glp = GitLabProvider(credentials=self.creds)
- glp.handle_webhook(self.repo, request_mock)
-
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.token, 'new-token')
- self.assertEqual(self.creds.refresh_token, 'new-refresh-token')
- self.assertEqual(self.creds.account_name, 'Someone')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
-
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_update_repo_references(self):
- """
- Check that we are able to fetch new branch and tags references
- """
- Ref = collections.namedtuple('Ref', 'name, commit')
-
- # Add 2 commits on 2 branches
- self.gl_mock().projects.get.return_value.branches.list.return_value = [
- Ref("master", {"id": "commit1", "message": "A commit on master", "author_email": "someone@teklia.com"}),
- # Create a commit with a very long branch name
- Ref("A" * 1000, {"id": "commit2", "message": "My fancy feature", "author_email": "another@teklia.com"})
- ]
-
- # Add 1 new commit on a tag, and reuse a commit on another tag
- self.gl_mock().projects.get.return_value.tags.list.return_value = [
- Ref("v0.1", {"id": "commit0", "message": "This is a legacy version", "author_email": "old@teklia.com"}),
- Ref("v1.0", {"id": "commit1", "message": "A commit on master", "author_email": "someone@teklia.com"})
- ]
-
- # Should only include data from fixtures at first
- self.assertFalse(self.repo.revisions.exclude(id=self.rev.id).exists())
- self.assertFalse(self.repo.refs.exists())
-
- # Update references for this repo
- provider = GitLabProvider(credentials=self.creds)
- provider.update_repository_references(self.repo)
-
- # We should now have 3 revisions
- commit0 = self.repo.revisions.get(hash="commit0")
- commit1 = self.repo.revisions.get(hash="commit1")
- commit2 = self.repo.revisions.get(hash="commit2")
- self.assertEqual(commit0.message, "This is a legacy version")
- self.assertEqual(commit1.message, "A commit on master")
- self.assertEqual(commit2.message, "My fancy feature")
- self.assertEqual(commit0.author, "old@teklia.com")
- self.assertEqual(commit1.author, "someone@teklia.com")
- self.assertEqual(commit2.author, "another@teklia.com")
-
- # Commit 1 should have a branch and a tag ref
- self.assertListEqual(list(commit1.refs.values_list('name', 'type').order_by('name')), [
- ('master', GitRefType.Branch),
- ('v1.0', GitRefType.Tag),
- ])
-
- # Other commits only have one ref
- self.assertListEqual(list(commit0.refs.values_list('name', 'type').order_by('name')), [
- ('v0.1', GitRefType.Tag),
- ])
- self.assertListEqual(list(commit2.refs.values_list('name', 'type').order_by('name')), [
- # Name has been restricted to 250 chars due to GitRef model constraint
- ('A' * 250, GitRefType.Branch),
- ])
-
- # We should now have 3 new processes for these refs
- self.assertListEqual(list(
- Process
- .objects
- .filter(mode=ProcessMode.Repository, revision__isnull=False)
- .values_list('name', 'mode', 'revision__hash')
- .order_by('revision__hash')
- ), [
- ("Import v0.1 from http://gitlab/repo", ProcessMode.Repository, 'commit0'),
- ("Import master from http://gitlab/repo", ProcessMode.Repository, 'commit1'),
- # Last process name has been limited to 100 chars due to Process model constraint
- (f"Import {'A' * 93}", ProcessMode.Repository, 'commit2'),
- ])
-
- @responses.activate
- @override_settings(PUBLIC_HOSTNAME='https://arkindex.localhost', GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t')
- def test_update_repo_references_refresh(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'refresh-token'
- }),
- ],
- json={
- 'access_token': 'new-token',
- 'refresh_token': 'new-refresh-token',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
- responses.get('https://somewhere/api/v4/user', json={'id': 42, 'username': 'Someone'})
- self.creds.expiry = None
- self.creds.save()
-
- self.gl_mock().projects.get.return_value.branches.list.return_value = []
- self.gl_mock().projects.get.return_value.tags.list.return_value = []
-
- provider = GitLabProvider(credentials=self.creds)
- provider.update_repository_references(self.repo)
-
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.token, 'new-token')
- self.assertEqual(self.creds.refresh_token, 'new-refresh-token')
- self.assertEqual(self.creds.account_name, 'Someone')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
diff --git a/arkindex/process/tests/test_process_datasets.py b/arkindex/process/tests/test_process_datasets.py
index a61ec306931425081bba986052fb16e75641b68d..2dce402d0caab3c71bc89db5cf2e1d48239c8f34 100644
--- a/arkindex/process/tests/test_process_datasets.py
+++ b/arkindex/process/tests/test_process_datasets.py
@@ -5,7 +5,7 @@ from django.urls import reverse
from rest_framework import status
from arkindex.documents.models import Corpus
-from arkindex.process.models import Process, ProcessDataset, ProcessMode
+from arkindex.process.models import Process, ProcessDataset, ProcessMode, Repository
from arkindex.project.tests import FixtureAPITestCase
from arkindex.training.models import Dataset
from arkindex.users.models import Role, User
@@ -48,8 +48,7 @@ class TestProcessDatasets(FixtureAPITestCase):
cls.dataset_process_2.datasets.set([cls.dataset2])
# For repository process
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
cls.repo.memberships.create(user=cls.test_user, level=Role.Admin.value)
cls.rev = cls.repo.revisions.get()
diff --git a/arkindex/process/tests/test_processes.py b/arkindex/process/tests/test_processes.py
index 80125798e18460ef51ec427c5b2b96e67986cd85..3a1a9f5f8b43d0a00fcdd03b134c8c3558b6da41 100644
--- a/arkindex/process/tests/test_processes.py
+++ b/arkindex/process/tests/test_processes.py
@@ -16,6 +16,7 @@ from arkindex.process.models import (
DataFile,
Process,
ProcessMode,
+ Repository,
WorkerActivity,
WorkerActivityState,
WorkerVersion,
@@ -38,8 +39,7 @@ class TestProcesses(FixtureAPITestCase):
cls.default_farm = Farm.objects.create(name='Corn farm')
cls.default_farm.memberships.create(user=cls.user, level=Role.Guest.value)
cls.other_farm = Farm.objects.get(name='Wheat farm')
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
cls.rev = cls.repo.revisions.get()
cls.dataset1, cls.dataset2 = Dataset.objects.filter(corpus=cls.corpus).order_by('name')
cls.private_corpus = Corpus.objects.create(name='Private corpus')
@@ -1851,23 +1851,6 @@ class TestProcesses(FixtureAPITestCase):
self.assertTrue(self.elts_process.tasks.exists())
self.assertFalse(delay_mock.called)
- def test_retry_repo_disabled(self):
- self.repository_process.revision = self.rev
- self.repository_process.save()
- self.repository_process.run()
- self.repository_process.tasks.all().update(state=State.Error)
- self.assertEqual(self.repository_process.state, State.Error)
- self.creds.delete()
- # Allow the user to do the retry
- self.repo.memberships.filter(user=self.user).update(level=Role.Admin.value)
- self.client.force_login(self.user)
-
- with self.assertNumQueries(10):
- response = self.client.post(reverse('api:process-retry', kwargs={'pk': self.repository_process.id}))
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {'__all__': ['Git repository does not have any valid credentials']})
-
@patch('arkindex.project.triggers.process_tasks.initialize_activity.delay')
def test_retry_initialize_activity(self, delay_mock):
"""
diff --git a/arkindex/process/tests/test_providers.py b/arkindex/process/tests/test_providers.py
deleted file mode 100644
index 0a6fb5324795d37c52850948ab43d86d1d784481..0000000000000000000000000000000000000000
--- a/arkindex/process/tests/test_providers.py
+++ /dev/null
@@ -1,125 +0,0 @@
-from unittest.mock import patch
-
-from django.core import mail
-from django.urls import reverse
-from rest_framework import status
-
-from arkindex.process.providers import GitLabProvider, GitProvider
-from arkindex.project.tests import FixtureAPITestCase
-
-MAIL_CONTENT = """
-Hello Test user,
-
-An error occurred while processing a new push on your repository http://my_repo.fake/workers/worker.
-
-Error: Error during handle_webhook
-
-No process will be launched to import your new revision.
-Please, try again with another push or contact your system administrator.
-
---
-Arkindex
-
-"""
-
-REFRESH_TOKEN_MAIL_CONTENT = """
-Hello Test user,
-
-An error occurred while processing a new push on your repository http://my_repo.fake/workers/worker.
-
-Error: The OAuth token could not be refreshed. Please reconnect the OAuth account manually.
-Original error: No refresh token was stored for this OAuth token.
-
-No process will be launched to import your new revision.
-Please, try again with another push or contact your system administrator.
-
---
-Arkindex
-
-"""
-
-
-class TestProviders(FixtureAPITestCase):
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
- cls.rev = cls.repo.revisions.get()
-
- def test_init(self):
- glp = GitLabProvider()
- self.assertIsNone(glp.credentials)
-
- glp = GitLabProvider(credentials=self.creds)
- self.assertEqual(glp.credentials, self.creds)
-
- with self.assertRaises(Exception):
- GitLabProvider(credentials='not a OAuthCredentials')
-
- @patch('arkindex.process.api.Repository.provider_class')
- def test_webhook_no_credentials(self, provider_class):
- self.client.force_login(self.user)
- self.repo.credentials = None
- self.repo.save()
- response = self.client.post(reverse('api:import-hook', kwargs={'pk': self.repo.id}))
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
- self.assertDictEqual(response.json(), {'detail': 'No credentials available for this repository.'})
- self.assertEqual(len(mail.outbox), 0)
-
- @patch('arkindex.process.api.Repository.provider_class')
- def test_webhook_handle_error(self, provider_class):
- provider_class.return_value.handle_webhook.side_effect = Exception('Error during handle_webhook')
- self.client.force_login(self.user)
- with self.assertRaisesRegex(Exception, 'Error during handle_webhook'):
- self.client.post(reverse('api:import-hook', kwargs={'pk': self.repo.id}))
- self.assertTrue(provider_class.return_value.handle_webhook.called)
- self.assertEqual(len(mail.outbox), 1)
- self.assertEqual(mail.outbox[0].to, ['user@user.fr'])
- self.assertEqual(mail.outbox[0].subject, 'An error occurred during Git hook handling')
- self.assertEqual(mail.outbox[0].body, MAIL_CONTENT)
-
- def test_webhook_handle_token_refresh_error(self):
- """
- The Git import hook should send an email for OAuthCredentials token refresh failures
- """
- self.client.force_login(self.user)
- self.creds.refresh_token = None
- self.creds.expiry = None
- self.assertTrue(self.creds.expired)
- self.creds.save()
-
- self.client.post(
- reverse('api:import-hook', kwargs={'pk': self.repo.id}),
- {
- 'object_kind': 'push',
- 'ref': 'refs/heads/something',
- 'commits': [],
- 'checkout_sha': 'abcd',
- },
- HTTP_X_GITLAB_EVENT='Push Hook',
- HTTP_X_GITLAB_TOKEN=self.repo.hook_token,
- )
-
- self.assertEqual(len(mail.outbox), 1)
- self.assertEqual(mail.outbox[0].to, ['user@user.fr'])
- self.assertEqual(mail.outbox[0].subject, 'An error occurred during Git hook handling')
- self.assertEqual(mail.outbox[0].body, REFRESH_TOKEN_MAIL_CONTENT)
-
- @patch('arkindex.process.api.Repository.provider_class')
- def test_webhook(self, provider_class):
- self.client.force_login(self.user)
- response = self.client.post(reverse('api:import-hook', kwargs={'pk': self.repo.id}))
- self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
- self.assertTrue(provider_class.return_value.handle_webhook.called)
- self.assertEqual(len(mail.outbox), 0)
-
- @patch('arkindex.process.models.Process.run')
- def test_start_imports_workers(self, run_mock):
- self.assertEqual(self.rev.processes.count(), 0)
- GitProvider.start_imports(None, rev=self.rev)
- self.assertEqual(self.rev.processes.count(), 1)
- self.assertEqual(run_mock.call_count, 1)
- _, run_kwargs = run_mock.call_args
- self.assertDictEqual(run_kwargs, {})
diff --git a/arkindex/process/tests/test_repos.py b/arkindex/process/tests/test_repos.py
index 57c84d4d6d8701de8bda32a70dcb210fbc3c5e03..46b9a22bf8608a5ab00c92cdabb3b3848c996c19 100644
--- a/arkindex/process/tests/test_repos.py
+++ b/arkindex/process/tests/test_repos.py
@@ -1,12 +1,9 @@
-from unittest.mock import MagicMock, patch
-
from django.urls import reverse
from rest_framework import status
-from rest_framework.exceptions import ValidationError
from rest_framework.serializers import DateTimeField
-from arkindex.ponos.models import Farm, State, Task
-from arkindex.process.models import ActivityState, Process, ProcessMode, Repository
+from arkindex.ponos.models import Farm
+from arkindex.process.models import ProcessMode, Repository
from arkindex.project.tests import FixtureTestCase
from arkindex.users.models import Role, User
@@ -16,9 +13,8 @@ class TestRepositories(FixtureTestCase):
@classmethod
def setUpTestData(cls):
super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
- cls.repo_2 = cls.creds.repos.get(url='http://gitlab/repo')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
+ cls.repo_2 = Repository.objects.get(url='http://gitlab/repo')
cls.rev = cls.repo.revisions.get()
cls.serialized_revision = {
'id': str(cls.rev.id),
@@ -35,15 +31,11 @@ class TestRepositories(FixtureTestCase):
return [
{
'id': str(self.repo_2.id),
- 'enabled': True,
- 'git_clone_url': None,
'url': self.repo_2.url,
'workers': [],
'authorized_users': self.repo_2.memberships.count(),
}, {
'id': str(self.repo.id),
- 'enabled': True,
- 'git_clone_url': None,
'url': self.repo.url,
'workers': [
{
@@ -57,35 +49,6 @@ class TestRepositories(FixtureTestCase):
}
]
- def test_delete_credentials_null(self):
- """
- Check deleting OAuthCredentials do not delete repositories and revisions
- """
- self.creds.delete()
- self.assertTrue(Repository.objects.filter(url='http://my_repo.fake/workers/worker').exists())
- self.repo.refresh_from_db()
- self.assertTrue(self.repo.revisions.exists())
-
- def test_no_credentials_no_process(self):
- """
- Check Repository import processes do not start without credentials
- """
- self.repo.credentials = None
- self.repo.save()
- task_count = Task.objects.count()
-
- process = Process.objects.create(
- mode=ProcessMode.Repository,
- revision=self.rev,
- creator=self.superuser,
- farm=Farm.objects.first(),
- )
-
- with self.assertRaisesMessage(ValidationError, 'Git repository does not have any valid credentials'):
- process.run()
-
- self.assertEqual(Task.objects.count(), task_count)
-
def test_list_repository_requires_login(self):
with self.assertNumQueries(0):
response = self.client.get(reverse('api:repository-list'))
@@ -157,8 +120,6 @@ class TestRepositories(FixtureTestCase):
data = response.json()
self.assertDictEqual(data, {
'id': str(self.repo_2.id),
- 'enabled': True,
- 'git_clone_url': None,
'url': self.repo_2.url,
'workers': [],
'authorized_users': self.repo_2.memberships.count(),
@@ -177,7 +138,7 @@ class TestRepositories(FixtureTestCase):
task = process.tasks.get()
self.repo_2.memberships.create(user=self.user, level=Role.Guest.value)
- with self.assertNumQueries(6):
+ with self.assertNumQueries(4):
response = self.client.get(
reverse('api:repository-retrieve', kwargs={'pk': str(self.repo_2.id)}),
HTTP_AUTHORIZATION=f'Ponos {task.token}',
@@ -186,8 +147,6 @@ class TestRepositories(FixtureTestCase):
self.assertDictEqual(response.json(), {
'id': str(self.repo_2.id),
- 'enabled': True,
- 'git_clone_url': 'https://oauth2:oauth-token@gitlab/repo',
'url': self.repo_2.url,
'workers': [],
'authorized_users': self.repo_2.memberships.count(),
@@ -206,7 +165,7 @@ class TestRepositories(FixtureTestCase):
task = process.tasks.get()
self.repo_2.memberships.create(user=self.user, level=Role.Guest.value)
- with self.assertNumQueries(6):
+ with self.assertNumQueries(4):
response = self.client.get(
reverse('api:repository-retrieve', kwargs={'pk': str(self.repo_2.id)}),
HTTP_AUTHORIZATION=f'Ponos {task.token}',
@@ -215,16 +174,12 @@ class TestRepositories(FixtureTestCase):
self.assertDictEqual(response.json(), {
'id': str(self.repo_2.id),
- 'enabled': True,
- 'git_clone_url': None,
'url': self.repo_2.url,
'workers': [],
'authorized_users': self.repo_2.memberships.count(),
})
def test_repository_retrieve_disabled_repo(self):
- self.repo_2.credentials = None
- self.repo_2.save()
self.repo_2.memberships.create(user=self.user, level=Role.Guest.value)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -233,8 +188,6 @@ class TestRepositories(FixtureTestCase):
data = response.json()
self.assertDictEqual(data, {
'id': str(self.repo_2.id),
- 'enabled': False,
- 'git_clone_url': None,
'url': self.repo_2.url,
'workers': [],
'authorized_users': self.repo_2.memberships.count(),
@@ -242,10 +195,10 @@ class TestRepositories(FixtureTestCase):
def test_repository_delete_not_admin(self):
"""
- A user cannot delete a repository if he has no admin access, even if the repository is based on its credentials
+ A user cannot delete a repository if he has no admin access
"""
creator = User.objects.create()
- repo = Repository.objects.create(credentials=creator.credentials.create(), url='http://somewhere.com/repo')
+ repo = Repository.objects.create(url='http://somewhere.com/repo')
repo.memberships.create(user=creator, level=Role.Contributor.value)
self.client.force_login(creator)
response = self.client.delete(reverse('api:repository-retrieve', kwargs={'pk': str(repo.id)}))
@@ -300,105 +253,3 @@ class TestRepositories(FixtureTestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json()
self.assertDictEqual(data, self.serialized_revision)
-
- @patch('arkindex.users.models.OAuthCredentials.git_provider_class')
- def test_list_repositories(self, git_provider_mock):
- repos = [MagicMock(id=i) for i in range(1, 4)]
- git_provider_mock().list_repos.return_value = repos
- git_provider_mock().to_dict = lambda repo: {
- 'id': repo.id,
- 'name': f'Repository {repo.id}',
- 'url': f'http://repo_{repo.id}'
- }
-
- self.client.force_login(self.user)
- response = self.client.get(
- reverse('api:available-repositories', kwargs={'pk': self.creds.id})
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(response.json(), [
- {'id': 1, 'name': 'Repository 1', 'url': 'http://repo_1'},
- {'id': 2, 'name': 'Repository 2', 'url': 'http://repo_2'},
- {'id': 3, 'name': 'Repository 3', 'url': 'http://repo_3'}
- ])
-
- def test_list_repositories_wrong_creds(self):
- self.client.force_login(self.user)
- response = self.client.get(
- reverse('api:available-repositories', kwargs={'pk': '12341234-1234-1234-1234-123412341234'})
- )
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
-
- def test_list_repositories_create_id_required(self):
- self.client.force_login(self.user)
- response = self.client.post(reverse('api:available-repositories', kwargs={'pk': self.creds.id}))
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
- self.assertEqual(response.json(), {'id': ['This field is required.']})
-
- @patch('arkindex.users.models.OAuthCredentials.git_provider_class')
- @patch('arkindex.process.models.Process.run')
- def test_list_repositories_create_workers_repo(self, process_run_mock, git_provider_mock):
- """
- Importing a workers repository creates a workers process
- """
- self.client.force_login(self.user)
- git_provider_mock().get_or_create_revision.return_value = self.rev, None
- repo = Repository.objects.create(url='http://somewhere.com/repo')
- git_provider_mock().create_repo.return_value = repo
-
- with self.assertNumQueries(15):
- response = self.client.post(reverse('api:available-repositories', kwargs={'pk': self.creds.id}), {'id': 1111})
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-
- process = Process.objects.get(revision=self.rev)
- self.assertEqual(response.json(), {
- 'name': 'Import 1337 from http://my_repo.fake/workers/worker',
- 'id': str(process.id),
- 'corpus': None,
- 'element': None,
- 'element_type': None,
- 'ml_class_id': None,
- 'load_children': False,
- 'use_cache': False,
- 'use_gpu': False,
- 'activity_state': ActivityState.Disabled.value,
- 'element_name_contains': None,
- 'files': [],
- 'folder_type': None,
- 'mode': ProcessMode.Repository.value,
- 'revision': self.serialized_revision,
- 'state': State.Unscheduled.value,
- 'template_id': None,
- 'model_id': None,
- 'train_folder_id': None,
- 'validation_folder_id': None,
- 'test_folder_id': None,
- })
-
- # Thumbnails are not generated for Workers processes
- self.assertEqual(process_run_mock.call_count, 1)
- _, kwargs = process_run_mock.call_args
- self.assertDictEqual(kwargs, {})
- self.assertIsNone(process.corpus)
- self.assertEqual(process.farm, self.farm)
-
- # User is granted an admin access to the repository
- repo_right = repo.memberships.get(user=self.user)
- self.assertTrue(repo_right.level >= Role.Admin.value)
-
- @patch('arkindex.users.models.OAuthCredentials.git_provider_class')
- def test_add_repository_farm_guest(self, git_provider_mock):
- self.client.force_login(self.user)
- git_provider_mock().get_or_create_revision.return_value = self.rev, None
- repo = Repository.objects.create(url='http://somewhere.com/repo')
- git_provider_mock().create_repo.return_value = repo
- self.farm.memberships.filter(user=self.user).delete()
-
- with self.assertNumQueries(12):
- response = self.client.post(reverse('api:available-repositories', kwargs={'pk': self.creds.id}), {'id': 1111})
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertListEqual(response.json(), [
- 'The owner of the OAuth credentials does not have access to the default farm.',
- ])
- self.assertFalse(self.rev.processes.exists())
diff --git a/arkindex/process/tests/test_signals.py b/arkindex/process/tests/test_signals.py
index 5b73f455ef16a9d73baf70fa28fe85167923d7b5..edb82f701fdc1fc963784d717aff7ef727549aa1 100644
--- a/arkindex/process/tests/test_signals.py
+++ b/arkindex/process/tests/test_signals.py
@@ -12,6 +12,7 @@ from arkindex.ponos.models import Farm, State
from arkindex.process.models import (
ActivityState,
ProcessMode,
+ Repository,
Worker,
WorkerActivityState,
WorkerRun,
@@ -29,8 +30,7 @@ class TestSignals(FixtureAPITestCase):
@classmethod
def setUpTestData(cls):
super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
cls.rev_1 = cls.repo.revisions.get()
cls.rev_2 = cls.repo.revisions.create(
hash='2',
diff --git a/arkindex/process/tests/test_worker_configurations.py b/arkindex/process/tests/test_worker_configurations.py
index a43e80f17d2f86c48e7c7860565c4376d7b999bb..519acc815723a5dc654e48b0d6e891bfd604a992 100644
--- a/arkindex/process/tests/test_worker_configurations.py
+++ b/arkindex/process/tests/test_worker_configurations.py
@@ -1,7 +1,7 @@
from django.urls import reverse
from rest_framework import status
-from arkindex.process.models import Revision, Worker
+from arkindex.process.models import Repository, Revision, Worker
from arkindex.project.tests import FixtureAPITestCase
from arkindex.users.models import Role
@@ -11,8 +11,7 @@ class TestWorkerConfigurations(FixtureAPITestCase):
@classmethod
def setUpTestData(cls):
super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
# User is the creator of the repository
cls.repo.memberships.create(user=cls.user, level=Role.Admin.value)
cls.rev = cls.repo.revisions.get()
diff --git a/arkindex/process/tests/test_workerruns.py b/arkindex/process/tests/test_workerruns.py
index cc08879e405276a4dbbef5e30d1f785b47ccbcd3..e4fa6e254afa8ffd3ca27ef4d30626d69bedecea 100644
--- a/arkindex/process/tests/test_workerruns.py
+++ b/arkindex/process/tests/test_workerruns.py
@@ -1,6 +1,5 @@
import uuid
from datetime import datetime, timezone
-from unittest.mock import patch
from django.test import override_settings
from django.urls import reverse
@@ -10,6 +9,7 @@ from rest_framework.exceptions import ValidationError
from arkindex.ponos.models import Agent, Artifact, Farm, State
from arkindex.process.models import (
FeatureUsage,
+ GitRef,
GitRefType,
ProcessMode,
Revision,
@@ -58,9 +58,6 @@ class TestWorkerRuns(FixtureAPITestCase):
# Add an execution access right on the worker
cls.worker_1.memberships.create(user=cls.user, level=Role.Contributor.value)
- cls.creds = cls.user.credentials.get()
- cls.gl_patch = patch('arkindex.process.providers.Gitlab')
-
# Model and Model version setup
cls.model_1 = Model.objects.create(name='My model')
cls.model_1.memberships.create(user=cls.user, level=Role.Contributor.value)
@@ -846,10 +843,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"""
Check the GitRefs are retrieved with the revision
"""
- from arkindex.process.providers import GitLabProvider
-
# Create gitrefs and check they were created
- self.gl_mock = self.gl_patch.start()
commit_refs = [
{'name': 'refs/tags/v0.1.0', 'type': 'tag'},
{'name': 'refs/heads/branch1', 'type': 'branch'},
@@ -863,8 +857,7 @@ class TestWorkerRuns(FixtureAPITestCase):
)
for ref in commit_refs:
- GitLabProvider(credentials=self.creds) \
- .update_or_create_ref(self.repo, revision, ref['name'], ref['type'])
+ GitRef.objects.create(repository=self.repo, revision=revision, name=ref['name'], type=ref['type'])
refs = [
{'name': ref.name, 'type': ref.type, 'repo': ref.repository}
diff --git a/arkindex/process/tests/test_workers.py b/arkindex/process/tests/test_workers.py
index b77525c42e4395d99887be965cc0de80fac0454f..fff0e0d05f5b2d1bc211cd9567d0e98f7ea69c1b 100644
--- a/arkindex/process/tests/test_workers.py
+++ b/arkindex/process/tests/test_workers.py
@@ -32,8 +32,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
@classmethod
def setUpTestData(cls):
super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+ cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
# User is the creator of the repository
cls.repo.memberships.create(user=cls.user, level=Role.Admin.value)
cls.rev = cls.repo.revisions.get()
@@ -436,9 +435,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
def test_workers_list_filter_repository(self):
repo2 = Repository.objects.create(
- url='http://gitlab/repo2',
- hook_token='hook-token2',
- credentials=self.creds,
+ url='http://gitlab/repo2'
)
worker_2 = Worker.objects.create(
repository=repo2,
@@ -478,9 +475,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
A user can retrieve a worker by having contributor access on its repository
"""
repo2 = Repository.objects.create(
- url='http://gitlab/repo2',
- hook_token='hook-token2',
- credentials=self.creds,
+ url='http://gitlab/repo2'
)
worker_2 = repo2.workers.create(name='Worker 2', slug='worker_2', type=self.worker_type_dla)
repo2.memberships.create(user=self.user, level=Role.Contributor.value)
@@ -1370,7 +1365,7 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
hash='1337',
message='A worker import I am admin on',
author='Yann',
- repo=self.creds.repos.get(url='http://gitlab/repo')
+ repo=Repository.objects.get(url='http://gitlab/repo')
)
with self.assertNumQueries(4):
diff --git a/arkindex/project/api_v1.py b/arkindex/project/api_v1.py
index e8d119d879cee29545e314fd39828e358925e612..fa57741099bb0ed914add56b844216d16c7664f9 100644
--- a/arkindex/project/api_v1.py
+++ b/arkindex/project/api_v1.py
@@ -77,7 +77,6 @@ from arkindex.ponos.api import (
)
from arkindex.process.api import (
ApplyProcessTemplate,
- AvailableRepositoriesList,
BucketList,
ClearProcess,
CorpusProcess,
@@ -89,7 +88,6 @@ from arkindex.process.api import (
DataFileList,
DataFileRetrieve,
FilesProcess,
- GitRepositoryImportHook,
ListProcessElements,
ProcessDatasetManage,
ProcessDatasets,
@@ -138,8 +136,6 @@ from arkindex.training.api import (
ValidateModelVersion,
)
from arkindex.users.api import (
- CredentialsList,
- CredentialsRetrieve,
GenericMembershipCreate,
GenericMembershipsList,
GroupDetails,
@@ -147,12 +143,8 @@ from arkindex.users.api import (
JobList,
JobRetrieve,
MembershipDetails,
- OAuthCallback,
- OAuthRetry,
- OAuthSignIn,
PasswordReset,
PasswordResetConfirm,
- ProvidersList,
UserCreate,
UserEmailLogin,
UserEmailVerification,
@@ -267,8 +259,6 @@ api = [
# Git import workflows
path('process/repos/', RepositoryList.as_view(), name='repository-list'),
path('process/repos/<uuid:pk>/', RepositoryRetrieve.as_view(), name='repository-retrieve'),
- path('process/repos/search/<uuid:pk>/', AvailableRepositoriesList.as_view(), name='available-repositories'),
- path('imports/hook/<uuid:pk>/', GitRepositoryImportHook.as_view(), name='import-hook'),
path('process/revisions/<uuid:pk>/', RevisionRetrieve.as_view(), name='revision-retrieve'),
# Workers
@@ -328,14 +318,6 @@ api = [
path('image/<uuid:pk>/', ImageRetrieve.as_view(), name='image-retrieve'),
path('image/<uuid:pk>/elements/', ImageElements.as_view(), name='image-elements'),
- # Manage OAuth integrations
- path('oauth/providers/', ProvidersList.as_view(), name='providers-list'),
- path('oauth/providers/<provider>/signin/', OAuthSignIn.as_view(), name='oauth-signin'),
- path('oauth/providers/<provider>/callback/', OAuthCallback.as_view(), name='oauth-callback'),
- path('oauth/credentials/', CredentialsList.as_view(), name='credentials-list'),
- path('oauth/credentials/<uuid:pk>/', CredentialsRetrieve.as_view(), name='credentials-retrieve'),
- path('oauth/credentials/<uuid:pk>/retry/', OAuthRetry.as_view(), name='oauth-retry'),
-
# Authentication
path('user/', UserRetrieve.as_view(), name='user-retrieve'),
path('user/new/', UserCreate.as_view(), name='user-new'),
diff --git a/arkindex/project/checks.py b/arkindex/project/checks.py
index 9a16474737878f89300d5814997348b7e4fa2a4f..c7205f95830116266edf8d8e622f303ce1ed7681 100644
--- a/arkindex/project/checks.py
+++ b/arkindex/project/checks.py
@@ -116,28 +116,6 @@ def ponos_env_check(*args, **kwargs):
return errors
-@register()
-@only_runserver
-def gitlab_oauth_check(*args, **kwargs):
- from django.conf import settings
- warnings = []
- app_id = getattr(settings, 'GITLAB_APP_ID', None)
- app_secret = getattr(settings, 'GITLAB_APP_SECRET', None)
- if not app_id:
- warnings.append(Warning(
- 'GitLab app ID is not set; Git imports will fail.',
- hint='settings.GITLAB_APP_ID = {}'.format(repr(app_id)),
- id='arkindex.W003',
- ))
- if not app_secret:
- warnings.append(Warning(
- 'GitLab app secret is not set; Git imports will fail.',
- hint='settings.GITLAB_APP_SECRET = {}'.format(repr(app_secret)),
- id='arkindex.W004',
- ))
- return warnings
-
-
@register()
@only_runserver
def s3_check(*args, **kwargs):
diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py
index 22867b41cb5afe585132368e266ec8ae6582a8fe..2e3eab3e7d18ac7ed09ff67d5831c1e941aff9ea 100644
--- a/arkindex/project/settings.py
+++ b/arkindex/project/settings.py
@@ -280,7 +280,6 @@ SPECTACULAR_SETTINGS = {
'description': 'Machine Learning models training',
},
{'name': 'datasets'},
- {'name': 'oauth'},
{'name': 'ponos'},
{'name': 'repos'},
{'name': 'search'},
@@ -310,9 +309,6 @@ IIIF_DOWNLOAD_TIMEOUT = (30, 60)
# check_images sample size when checking all servers
CHECK_IMAGES_SAMPLE_SIZE = 20
-# GitLab OAuth
-GITLAB_APP_ID = conf['gitlab']['app_id']
-GITLAB_APP_SECRET = conf['gitlab']['app_secret']
if conf['cache']['type'] is None:
conf['cache']['type'] = CacheType.Dummy if DEBUG else CacheType.Memory
@@ -586,12 +582,6 @@ LICENSE_KEY = conf['license']['key']
LICENSE_PING_FREQUENCY = conf['license']['ping_frequency']
LICENSE_HOSTNAME = conf['license']['hostname']
-# Specific setting to enable GitLab webhook when using PageKite, it should not be used in production.
-# It is used to change GitLab webhook URLs to go through a specified PageKite domain instead of localhost.
-# None by default.
-# You can override its value through local_settings.py file by assigning it 'http://YOURDOMAIN.pagekite.me'
-BACKEND_PUBLIC_URL_OAUTH = None
-
# Optional default group to add new users to when they complete email verification
SIGNUP_DEFAULT_GROUP = conf['signup_default_group']
diff --git a/arkindex/project/tests/test_checks.py b/arkindex/project/tests/test_checks.py
index 222238e8b90c639ce5c3f47ed35cbac603eef832..214be549bec8bfb5d663ebd1c38d9ed614ba94cf 100644
--- a/arkindex/project/tests/test_checks.py
+++ b/arkindex/project/tests/test_checks.py
@@ -88,31 +88,6 @@ class ChecksTestCase(TestCase):
),
])
- @override_settings()
- def test_gitlab_oauth_check(self):
- from arkindex.project.checks import gitlab_oauth_check
-
- del settings.GITLAB_APP_ID
- del settings.GITLAB_APP_SECRET
-
- self.assertListEqual(gitlab_oauth_check(), [
- Warning(
- 'GitLab app ID is not set; Git imports will fail.',
- hint='settings.GITLAB_APP_ID = None',
- id='arkindex.W003',
- ),
- Warning(
- 'GitLab app secret is not set; Git imports will fail.',
- hint='settings.GITLAB_APP_SECRET = None',
- id='arkindex.W004',
- ),
- ])
-
- settings.GITLAB_APP_ID = '1234'
- settings.GITLAB_APP_SECRET = 's3kr3t'
-
- self.assertListEqual(gitlab_oauth_check(), [])
-
@override_settings()
def test_s3_check(self):
from arkindex.project.checks import s3_check
diff --git a/arkindex/project/urls.py b/arkindex/project/urls.py
index de393c87a537550d9079297bc81e3af0b46014e9..7ecaae664800222d763829341b3b7df6ca98b09e 100644
--- a/arkindex/project/urls.py
+++ b/arkindex/project/urls.py
@@ -19,8 +19,6 @@ urlpatterns = [
# Frontend URLs the backend needs with django.urls.reverse
# Link sent via email for password resets
path('user/reset/<uidb64>/<token>/', frontend_view.as_view(), name='password_reset_confirm'),
- # Redirection URL for successful OAuth2 flows
- path('process/credentials/', frontend_view.as_view(), name='credentials'),
path('process/<uuid:pk>/<int:run>', frontend_view.as_view(), name='frontend-process-details'),
# Link to the corpus management page, shown in the Django admin
path('corpus/<uuid:pk>', frontend_view.as_view(), name='frontend-corpus-details'),
diff --git a/arkindex/users/admin.py b/arkindex/users/admin.py
index aac4399da50225f15ece65e24a2f917000966f4b..74a14398a9fc7bb6930e3e73e11848ba52e464a8 100644
--- a/arkindex/users/admin.py
+++ b/arkindex/users/admin.py
@@ -7,7 +7,7 @@ from django.contrib.contenttypes.admin import GenericTabularInline
from django.db.models.functions import Collate
from enumfields.admin import EnumFieldListFilter
-from arkindex.users.models import Group, OAuthCredentials, Right, User, UserScope
+from arkindex.users.models import Group, Right, User, UserScope
class UserCreationForm(forms.ModelForm):
@@ -122,25 +122,7 @@ class GroupAdmin(admin.ModelAdmin):
inlines = (UserMembershipInline, )
-class OAuthCredentialAdmin(admin.ModelAdmin):
- list_display = ('id', 'user', 'status')
- list_filter = [('status', EnumFieldListFilter), ]
- fields = ('id', 'user', 'token', 'refresh_token', 'expiry', 'status')
- readonly_fields = ('id', )
- search_fields = ('searchable_email', )
-
- def get_queryset(self, request):
- qs = super().get_queryset(request)
- qs = qs.select_related('user')
- # Django's search uses ILIKE, but LIKE is not supported in PostgreSQL with nondeterministic collations.
- # We are using a case-insensitive collation to get unique case-insensitive emails, so to search properly,
- # we first have to set the collation back to its default value, making it case-sensitive again;
- # since Django uses ILIKE, it will be case-insensitive anyway.
- return qs.annotate(searchable_email=Collate('user__email', 'default'))
-
-
admin.site.register(User, UserAdmin)
-admin.site.register(OAuthCredentials, OAuthCredentialAdmin)
# Register the custom GroupAdmin
admin.site.register(Group, GroupAdmin)
# and hide base GroupAdmin form contrib.auth
diff --git a/arkindex/users/api.py b/arkindex/users/api.py
index 8cb07fe06ea27bd1b89455821abd4c037b85598a..2cfb72e25b57d75ed03492f6d44419b9222b99c9 100644
--- a/arkindex/users/api.py
+++ b/arkindex/users/api.py
@@ -5,7 +5,6 @@ from uuid import UUID
from django.conf import settings
from django.contrib.auth import login, logout
from django.contrib.auth.forms import PasswordResetForm
-from django.contrib.auth.mixins import UserPassesTestMixin
from django.contrib.auth.tokens import default_token_generator
from django.contrib.contenttypes.models import ContentType
from django.core.mail import send_mail
@@ -14,18 +13,16 @@ from django.shortcuts import get_object_or_404
from django.template.loader import render_to_string
from django.urls import reverse
from django.utils.http import urlsafe_base64_encode
-from django.views.generic import RedirectView
from django_rq.queues import get_queue
from django_rq.settings import QUEUES
from drf_spectacular.types import OpenApiTypes
-from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view, inline_serializer
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
from rest_framework import serializers, status
from rest_framework.exceptions import AuthenticationFailed, NotFound, PermissionDenied, ValidationError
from rest_framework.generics import (
CreateAPIView,
ListAPIView,
ListCreateAPIView,
- RetrieveAPIView,
RetrieveDestroyAPIView,
RetrieveUpdateDestroyAPIView,
)
@@ -38,8 +35,7 @@ from arkindex.documents.models import Corpus
from arkindex.process.models import Worker
from arkindex.project.mixins import ACLMixin, WorkerACLMixin
from arkindex.project.permissions import IsAuthenticatedOrReadOnly, IsVerified
-from arkindex.users import providers
-from arkindex.users.models import Group, OAuthCredentials, OAuthStatus, Right, Role, Scope, User, UserScope
+from arkindex.users.models import Group, Right, Role, Scope, User, UserScope
from arkindex.users.serializers import (
EmailLoginSerializer,
GenericMembershipSerializer,
@@ -49,9 +45,6 @@ from arkindex.users.serializers import (
MembershipCreateSerializer,
MembershipSerializer,
NewUserSerializer,
- OAuthCredentialsSerializer,
- OAuthProviderClassSerializer,
- OAuthRetrySerializer,
PasswordResetConfirmSerializer,
PasswordResetSerializer,
UserSerializer,
@@ -61,64 +54,6 @@ from arkindex.users.utils import RightContent, get_max_level
logger = logging.getLogger(__name__)
-@extend_schema_view(get=extend_schema(operation_id='ListOAuthProviders', tags=['oauth']))
-class ProvidersList(ListAPIView):
- """
- List supported OAuth providers
- """
- permission_classes = (IsVerified, )
- serializer_class = OAuthProviderClassSerializer
- pagination_class = None
-
- def get_queryset(self):
- return list(filter(lambda p: p.enabled(), providers.oauth_providers))
-
-
-@extend_schema(
- tags=['oauth'],
- parameters=[
- OpenApiParameter(
- 'status',
- description='Filter OAuth credentials by their status',
- required=False,
- enum=[status.value for status in OAuthStatus],
- )
- ]
-)
-class CredentialsList(ListAPIView):
- """
- List all OAuth credentials for the authenticated user.
- """
- permission_classes = (IsVerified, )
- serializer_class = OAuthCredentialsSerializer
-
- def get_queryset(self):
- qs = self.request.user.credentials.order_by('id')
- if 'status' in self.request.query_params:
- qs = qs.filter(status=self.request.query_params['status'])
- return qs
-
-
-@extend_schema(tags=['oauth'])
-@extend_schema_view(
- delete=extend_schema(description='Delete OAuth credentials. This may disable access to some Git repositories.')
-)
-class CredentialsRetrieve(RetrieveDestroyAPIView):
- """
- Retrieve OAuth credentials.
- """
- permission_classes = (IsVerified, )
- serializer_class = OAuthCredentialsSerializer
- queryset = OAuthCredentials.objects.none()
-
- def get_queryset(self):
- return self.request.user.credentials.order_by('id')
-
- def perform_destroy(self, instance):
- instance.provider_class(credentials=instance).disconnect()
- super().perform_destroy(instance)
-
-
@extend_schema(tags=['users'])
@extend_schema_view(
get=extend_schema(description='Retrieve information about the authenticated user'),
@@ -355,114 +290,6 @@ class PasswordResetConfirm(CreateAPIView):
serializer_class = PasswordResetConfirmSerializer
-class OAuthSignIn(APIView):
- """
- Start the OAuth authentication code flow for a given provider
- """
- permission_classes = (IsVerified, )
- queryset = OAuthCredentials.objects.none()
-
- @extend_schema(
- operation_id='StartOAuthSignIn',
- tags=['oauth'],
- parameters=[
- OpenApiParameter(
- 'provider',
- location=OpenApiParameter.PATH,
- required=True,
- description='Provider name',
- )
- ],
- responses=inline_serializer(
- name='OAuthSignInResponse',
- fields={'url': serializers.URLField(required=False, help_text="URL to the authorization endpoint.")}
- )
- )
- def get(self, *args, **kwargs):
- if 'provider' not in kwargs:
- raise ValidationError('Missing provider')
-
- provider_class = providers.get_provider(kwargs['provider'])
- if not provider_class:
- raise ValidationError('Unknown provider')
-
- url = self.request.GET.get('url', provider_class.default_url)
- if url:
- parsed = urllib.parse.urlparse(url)
- if (parsed.scheme != 'https'
- or parsed.netloc == ''
- or parsed.params != ''
- or parsed.query != ''
- or parsed.fragment != ''):
- raise ValidationError('Invalid GitLab instance URL')
-
- # Create OAuthCredentials without a token
- creds = self.request.user.credentials.create(
- provider_url=url,
- )
-
- return Response({
- 'url': provider_class(credentials=creds).get_authorize_uri(),
- })
-
-
-@extend_schema_view(get=extend_schema(operation_id='RetryOAuthCredentials', tags=['oauth']))
-class OAuthRetry(RetrieveAPIView):
- """
- Retry the OAuth authentication code flow for pending credentials
- """
- permission_classes = (IsVerified, )
- serializer_class = OAuthRetrySerializer
- queryset = OAuthCredentials.objects.none()
-
- def get_queryset(self):
- return self.request.user.credentials
-
- def get_object(self):
- creds = super().get_object()
- if creds.status == OAuthStatus.Done:
- self.permission_denied()
- return creds
-
- def retrieve(self, request, *args, **kwargs):
- creds = self.get_object()
- return Response({
- 'url': creds.provider_class(credentials=creds).get_authorize_uri(),
- })
-
-
-class OAuthCallback(UserPassesTestMixin, RedirectView):
- """
- Callback for OAuth responses
- """
- pattern_name = 'credentials'
- permission_classes = (IsVerified, )
- raise_exception = True
-
- def test_func(self):
- """
- Perform Django REST framework permission checks, but without REST framework.
- """
- return all(perm().has_permission(self.request, self) for perm in self.permission_classes)
-
- def get(self, request, *args, **kwargs):
- assert 'provider' in kwargs
- provider_class = providers.get_provider(kwargs['provider'])
- if not provider_class:
- raise ValueError('Unknown provider')
- provider = provider_class()
- try:
- provider.handle_callback(request)
- provider.credentials.status = OAuthStatus.Done
- except Exception as e:
- if not provider.credentials:
- raise
- provider.credentials.status = OAuthStatus.Error
- logger.warning(f'OAuth callback error: {e}', exc_info=e)
- provider.credentials.save()
- return super().get(request)
-
-
@extend_schema(tags=['users'])
class GroupsList(ListCreateAPIView):
"""
diff --git a/arkindex/users/migrations/0001_initial.py b/arkindex/users/migrations/0001_initial.py
index f09617397f666eddc783a58aa24b3385cf1e5eef..8dc6ec7c91d5a901bf65dbb8841a1bff6604f496 100644
--- a/arkindex/users/migrations/0001_initial.py
+++ b/arkindex/users/migrations/0001_initial.py
@@ -8,10 +8,17 @@ import enumfields.fields
from django.conf import settings
from django.contrib.postgres.operations import CreateCollation
from django.db import migrations, models
+from enumfields import Enum
import arkindex.users.models
+class OAuthStatus(Enum):
+ Created = 'created'
+ Done = 'done'
+ Error = 'error'
+
+
class Migration(migrations.Migration):
initial = True
@@ -70,7 +77,7 @@ class Migration(migrations.Migration):
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('provider_url', models.URLField()),
- ('status', enumfields.fields.EnumField(default='created', enum=arkindex.users.models.OAuthStatus, max_length=10)),
+ ('status', enumfields.fields.EnumField(default='created', enum=OAuthStatus, max_length=10)),
('token', models.CharField(blank=True, max_length=64, null=True)),
('refresh_token', models.CharField(blank=True, max_length=64, null=True)),
('expiry', models.DateTimeField(blank=True, null=True)),
diff --git a/arkindex/users/migrations/0003_delete_oauthcredentials.py b/arkindex/users/migrations/0003_delete_oauthcredentials.py
new file mode 100644
index 0000000000000000000000000000000000000000..d124d8dba938962c9b821fd8d445bb9084516ad2
--- /dev/null
+++ b/arkindex/users/migrations/0003_delete_oauthcredentials.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-12-20 11:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('process', '0026_remove_repository_unique_repository_hook_token_and_more'),
+ ('users', '0002_remove_user_transkribus_email'),
+ ]
+
+ operations = [
+ migrations.DeleteModel(
+ name='OAuthCredentials',
+ ),
+ ]
diff --git a/arkindex/users/models.py b/arkindex/users/models.py
index 3764f14854abf7be672a32b30d39380702c30d51..db686b0494c25536cfc57430677cc82482cba997 100644
--- a/arkindex/users/models.py
+++ b/arkindex/users/models.py
@@ -1,5 +1,4 @@
import uuid
-from datetime import datetime, timezone
from django.conf import settings
from django.contrib.auth.models import AbstractBaseUser
@@ -12,7 +11,6 @@ from enumfields import Enum, EnumField
from rq.utils import as_text
from arkindex.users.managers import UserManager
-from arkindex.users.providers import get_provider
class Role(Enum):
@@ -186,55 +184,6 @@ class Group(models.Model):
return self.name
-class OAuthStatus(Enum):
- Created = 'created'
- Done = 'done'
- Error = 'error'
-
-
-class OAuthCredentials(models.Model):
- id = models.UUIDField(default=uuid.uuid4, primary_key=True, editable=False)
- user = models.ForeignKey('users.User', on_delete=models.CASCADE, related_name='credentials')
- provider_url = models.URLField()
- status = EnumField(OAuthStatus, default=OAuthStatus.Created)
- token = models.CharField(max_length=64, blank=True, null=True)
- refresh_token = models.CharField(max_length=64, blank=True, null=True)
- expiry = models.DateTimeField(blank=True, null=True)
- account_name = models.CharField(max_length=100, blank=True, null=True)
-
- def __str__(self):
- token_str = f"- {self.token[:6]}" if self.token else ''
- return f"{self.provider_name} - {self.user.email} {token_str}".strip()
-
- # Make this constant into a database field to be able to handle OAuth with services other than GitLab
- provider_name = 'gitlab'
-
- @property
- def provider_class(self):
- return get_provider(self.provider_name)
-
- @property
- def provider(self):
- return self.provider_class(credentials=self)
-
- @property
- def git_provider_class(self):
- from arkindex.process.providers import from_oauth
- return from_oauth(self.provider_name)
-
- @property
- def git_provider(self):
- return self.git_provider_class(credentials=self)
-
- @property
- def expired(self):
- return self.expiry is None or self.expiry < datetime.now(timezone.utc)
-
- class Meta:
- verbose_name = 'OAuth credentials'
- verbose_name_plural = 'OAuth credentials'
-
-
class Scope(Enum):
UploadS3Image = 'upload_s3_image'
diff --git a/arkindex/users/providers.py b/arkindex/users/providers.py
deleted file mode 100644
index c30fcb2367013f9c952a48331bfbe3a8e2a98f3d..0000000000000000000000000000000000000000
--- a/arkindex/users/providers.py
+++ /dev/null
@@ -1,199 +0,0 @@
-import urllib.parse
-from abc import ABC, abstractmethod
-from datetime import datetime, timedelta, timezone
-
-import requests
-from django.conf import settings
-from django.urls import reverse
-from gitlab import Gitlab, GitlabError
-from rest_framework.exceptions import AuthenticationFailed, NotAuthenticated
-
-
-class OAuthProvider(ABC):
- """
- An OAuth authentication provider.
- """
-
- display_name = ""
- slug = ""
-
- def __init__(self, credentials=None, url=None):
- if credentials is not None:
- from arkindex.users.models import OAuthCredentials
- assert isinstance(credentials, OAuthCredentials)
- self.credentials = credentials
-
- @classmethod
- @abstractmethod
- def enabled(cls):
- """
- Boolean stating the provider's availability.
- """
-
- @abstractmethod
- def get_callback_uri(self):
- """
- Get the OAuth callback URI
- """
-
- @abstractmethod
- def get_authorize_uri(self):
- """
- Get the OAuth authorization endpoint URI
- """
-
- @abstractmethod
- def handle_callback(self, request):
- """
- Handle a OAuth callback and save token data. Should raise exceptions if the process fails.
- """
-
- @abstractmethod
- def refresh_token(self):
- """
- Refresh an expired OAuth token and update OAuthCredentials attributes.
- """
-
- @abstractmethod
- def disconnect(self):
- """
- Erase token data and logout the user from the service.
- """
-
-
-class GitLabOAuthProvider(OAuthProvider):
-
- display_name = 'GitLab'
- slug = 'gitlab'
- default_url = 'https://gitlab.com'
- authorize_endpoint = '/oauth/authorize'
- token_endpoint = '/oauth/token'
-
- @classmethod
- def enabled(cls):
- return settings.GITLAB_APP_ID and settings.GITLAB_APP_SECRET
-
- def get_callback_uri(self):
- url = reverse('api:oauth-callback', kwargs={'provider': self.slug})
-
- if settings.BACKEND_PUBLIC_URL_OAUTH:
- return urllib.parse.urljoin(settings.BACKEND_PUBLIC_URL_OAUTH, url)
-
- assert settings.PUBLIC_HOSTNAME, 'PUBLIC_HOSTNAME is required to generate callback URIs'
- return urllib.parse.urljoin(settings.PUBLIC_HOSTNAME, url)
-
- def get_authorize_uri(self):
- if not self.credentials:
- return
-
- return '{}?{}'.format(
- urllib.parse.urljoin(self.credentials.provider_url, self.authorize_endpoint),
- urllib.parse.urlencode({
- 'client_id': settings.GITLAB_APP_ID,
- 'redirect_uri': self.get_callback_uri(),
- 'scope': 'api',
- 'response_type': 'code',
- 'state': str(self.credentials.id),
- }),
- )
-
- def handle_callback(self, request):
- state = request.GET.get('state')
- if not state:
- raise ValueError('No state hash')
-
- self.credentials = request.user.credentials.get(id=state)
-
- if not any(param in request.GET for param in ('code', 'error')):
- raise ValueError('Callback called without a valid response')
- if 'error' in request.GET:
- raise ValueError(request.GET.get('error_description', request.GET['error']))
-
- from arkindex.users.models import OAuthStatus
- assert self.credentials.status != OAuthStatus.Done, 'Cannot overwrite existing credentials'
-
- self.grant_token(grant_type='authorization_code', code=request.GET.get('code', ''))
-
- def refresh_token(self):
- try:
- assert self.credentials.refresh_token, 'No refresh token was stored for this OAuth token.'
- self.grant_token(grant_type='refresh_token', refresh_token=self.credentials.refresh_token)
- except Exception as e:
- # Set an error state to allow the user to retry manually
- from arkindex.users.models import OAuthStatus
- self.credentials.status = OAuthStatus.Error
- self.credentials.save()
- raise AuthenticationFailed(
- 'The OAuth token could not be refreshed. Please reconnect the OAuth account manually.\n'
- f'Original error: {e}'
- )
-
- def grant_token(self, **kwargs):
- """
- Use Doorkeeper's OAuth token endpoint to get a token, either from an OAuth authorization code flow
- or when refreshing a token, and update the OAuthCredentials attributes.
-
- https://github.com/doorkeeper-gem/doorkeeper/wiki/API-endpoint-descriptions-and-examples#post---oauthtoken
- """
- assert self.credentials is not None, 'An OAuthCredentials instance is required to use the OAuth token endpoint'
-
- payload = {
- 'client_id': settings.GITLAB_APP_ID,
- 'client_secret': settings.GITLAB_APP_SECRET,
- 'redirect_uri': self.get_callback_uri(),
- }
- payload.update(kwargs)
- response = requests.post(
- urllib.parse.urljoin(self.credentials.provider_url, self.token_endpoint),
- payload,
- )
- response.raise_for_status()
- data = response.json()
-
- assert 'access_token' in data, 'GitLab returned no OAuth token'
- assert data.get('token_type', '').lower() == 'bearer', 'Unsupported OAuth token type'
-
- self.credentials.token = data['access_token']
- self.credentials.refresh_token = data.get('refresh_token')
- # Get absolute date of expiration from the created_at UNIX timestamp and the seconds until expiration
- self.credentials.expiry = datetime.fromtimestamp(data['created_at'], tz=timezone.utc) + timedelta(seconds=int(data['expires_in']))
-
- gl = Gitlab(self.credentials.provider_url, oauth_token=self.credentials.token)
- gl.auth()
- self.credentials.account_name = gl.user.username
-
- from arkindex.users.models import OAuthStatus
- self.credentials.status = OAuthStatus.Done
- self.credentials.save()
-
- def disconnect(self):
- if not self.credentials:
- raise NotAuthenticated
-
- if not self.credentials.token or not self.credentials.repos.exists():
- return
-
- # Remove all webhooks
- try:
- gl = Gitlab(self.credentials.provider_url, oauth_token=self.credentials.token)
- for repo in self.credentials.repos.all():
- project = gl.projects.get(urllib.parse.urlsplit(repo.url).path.strip('/'))
- hook_url = urllib.parse.urljoin(
- settings.PUBLIC_HOSTNAME,
- reverse('api:import-hook', kwargs={'pk': repo.id})
- )
- # Try to find the webhook
- hook = next((h for h in project.hooks.list() if h.url == hook_url), None)
- if hook:
- hook.delete()
- except GitlabError:
- pass
-
-
-oauth_providers = [
- GitLabOAuthProvider,
-]
-
-
-def get_provider(name):
- return next(filter(lambda p: p.slug == name, oauth_providers), None)
diff --git a/arkindex/users/serializers.py b/arkindex/users/serializers.py
index 839c8696f71a3f9b1b6ff5abab86fe55ff449174..283124e5da19e2bbf4d384f7aa4c0729c548c984 100644
--- a/arkindex/users/serializers.py
+++ b/arkindex/users/serializers.py
@@ -11,8 +11,7 @@ from rest_framework.exceptions import PermissionDenied
from arkindex.process.models import Worker
from arkindex.project.mixins import WorkerACLMixin
-from arkindex.project.serializer_fields import EnumField
-from arkindex.users.models import Group, OAuthCredentials, OAuthStatus, Right, Role, User
+from arkindex.users.models import Group, Right, Role, User
from arkindex.users.utils import RightContent, get_max_level
@@ -26,40 +25,6 @@ def validate_user_password(user, data):
return data
-class OAuthCredentialsSerializer(serializers.ModelSerializer):
-
- provider_name = serializers.CharField()
- provider_display_name = serializers.CharField(source='provider_class.display_name')
- status = EnumField(OAuthStatus)
-
- class Meta:
- model = OAuthCredentials
- fields = (
- 'id',
- 'status',
- 'provider_name',
- 'provider_display_name',
- 'provider_url',
- 'account_name',
- )
-
-
-class OAuthProviderClassSerializer(serializers.Serializer):
-
- name = serializers.CharField(source='slug')
- display_name = serializers.CharField()
- default_url = serializers.URLField()
-
-
-class OAuthRetrySerializer(serializers.Serializer):
- """
- A serializer used by the OAuthRetry endpoint to return an authorization URL.
- Required to get the OpenAPI schema generation to work.
- """
-
- url = serializers.URLField()
-
-
class SimpleUserSerializer(serializers.ModelSerializer):
class Meta:
diff --git a/arkindex/users/tests/test_credentials.py b/arkindex/users/tests/test_credentials.py
deleted file mode 100644
index 5669c7c7d3e5a5f96faa53802a39b403df9cb6ae..0000000000000000000000000000000000000000
--- a/arkindex/users/tests/test_credentials.py
+++ /dev/null
@@ -1,146 +0,0 @@
-import uuid
-
-from django.urls import reverse
-from rest_framework import status
-
-from arkindex.project.tests import FixtureTestCase
-from arkindex.users.models import OAuthCredentials, OAuthStatus, User
-
-
-class TestCredentials(FixtureTestCase):
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.cred1 = cls.user.credentials.get()
- cls.cred2 = cls.user.credentials.create(
- provider_url='https://somewhere.else',
- token='t0k3n',
- status=OAuthStatus.Error,
- )
- cls.user2 = User.objects.create(email='user2@test.test', display_name='User 2', verified_email=True)
- cls.cred3 = cls.user2.credentials.create(
- provider_url='https://somewhere.else',
- token='t0k3n',
- status=OAuthStatus.Created,
- )
-
- def test_list_requires_login(self):
- response = self.client.get(reverse('api:credentials-list'))
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-
- def test_list(self):
- self.client.force_login(self.user)
- response = self.client.get(reverse('api:credentials-list'))
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- data = response.json()
- self.assertCountEqual(data['results'], [
- {
- 'id': str(self.cred1.id),
- 'status': 'created',
- 'provider_name': 'gitlab',
- 'provider_display_name': 'GitLab',
- 'provider_url': 'https://somewhere',
- 'account_name': None,
- },
- {
- 'id': str(self.cred2.id),
- 'status': 'error',
- 'provider_name': 'gitlab',
- 'provider_display_name': 'GitLab',
- 'provider_url': 'https://somewhere.else',
- 'account_name': None,
- },
- ])
-
- def test_list_filtered(self):
- self.client.force_login(self.user)
- response = self.client.get(reverse('api:credentials-list'), {'status': 'error'})
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- data = response.json()
- self.assertCountEqual(data['results'], [
- {
- 'id': str(self.cred2.id),
- 'status': 'error',
- 'provider_name': 'gitlab',
- 'provider_display_name': 'GitLab',
- 'provider_url': 'https://somewhere.else',
- 'account_name': None,
- },
- ])
-
- def test_credentials_retrieve_requires_login(self):
- with self.assertNumQueries(0):
- response = self.client.get(reverse('api:credentials-retrieve', kwargs={'pk': str(self.cred1.id)}))
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-
- def test_credentials_retrieve_not_found(self):
- fake_id = uuid.uuid4()
- self.client.force_login(self.user)
- with self.assertNumQueries(3):
- response = self.client.get(reverse('api:credentials-retrieve', kwargs={'pk': str(fake_id)}))
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
-
- def test_credentials_retrieve(self):
- self.client.force_login(self.user)
- with self.assertNumQueries(3):
- response = self.client.get(reverse('api:credentials-retrieve', kwargs={'pk': str(self.cred1.id)}))
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertDictEqual(response.json(), {
- 'id': str(self.cred1.id),
- 'status': 'created',
- 'provider_name': 'gitlab',
- 'provider_display_name': 'GitLab',
- 'provider_url': 'https://somewhere',
- 'account_name': None,
- })
-
- def test_credentials_retrieve_wrong_user(self):
- """
- A user cannot retrieve another user's credentials
- """
- self.client.force_login(self.user)
- with self.assertNumQueries(3):
- response = self.client.get(reverse('api:credentials-retrieve', kwargs={'pk': str(self.cred3.id)}))
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
-
- def test_credentials_display_with_token(self):
- self.assertEqual(str(self.cred1), 'gitlab - user@user.fr - oauth-')
-
- def test_credentials_display_no_token(self):
- no_token = self.user.credentials.create(
- provider_url='https://somewhere.else',
- status=OAuthStatus.Created,
- )
- self.assertEqual(str(no_token), 'gitlab - user@user.fr')
-
- def test_destroy_credentials_requires_logged_in(self):
- with self.assertNumQueries(0):
- response = self.client.delete(reverse('api:credentials-retrieve', kwargs={"pk": str(self.cred2.id)}))
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
- self.assertDictEqual(response.json(), {'detail': 'Authentication credentials were not provided.'})
-
- def test_destroy_credentials_requires_verified(self):
- self.user.verified_email = False
- self.user.save()
- self.client.force_login(self.user)
- with self.assertNumQueries(2):
- response = self.client.delete(reverse('api:credentials-retrieve', kwargs={"pk": str(self.cred2.id)}))
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
- self.assertDictEqual(response.json(), {'detail': 'You do not have permission to perform this action.'})
-
- def test_destroy_credentials(self):
- self.client.force_login(self.user)
- with self.assertNumQueries(6):
- response = self.client.delete(reverse('api:credentials-retrieve', kwargs={"pk": str(self.cred2.id)}))
- self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
-
- # Make sure it does not exist anymore
- with self.assertRaises(OAuthCredentials.DoesNotExist):
- self.cred2.refresh_from_db()
-
- def test_destroy_credentials_not_found(self):
- self.client.force_login(self.user)
- with self.assertNumQueries(3):
- response = self.client.delete(reverse('api:credentials-retrieve', kwargs={"pk": str(uuid.uuid4())}))
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
diff --git a/arkindex/users/tests/test_generic_memberships.py b/arkindex/users/tests/test_generic_memberships.py
index 172789d620bb3edca8ddd5a17362aeb5adca7ad6..d35a130ee58169353d1e1e9638c14b0eb1c9887e 100644
--- a/arkindex/users/tests/test_generic_memberships.py
+++ b/arkindex/users/tests/test_generic_memberships.py
@@ -289,7 +289,7 @@ class TestMembership(FixtureAPITestCase):
"""
A user is able to list members of a worker if he is a member of its repository
"""
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
worker = repo.workers.get(slug='reco')
repo.memberships.create(user=self.user, level=Role.Guest.value)
self.client.force_login(self.user)
@@ -784,7 +784,7 @@ class TestMembership(FixtureAPITestCase):
"""
user = User.objects.create_user('test@test.de', 'Pa$$w0rd')
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
worker = repo.workers.get(slug='reco')
repo.memberships.create(user=self.user, level=Role.Admin.value)
@@ -870,7 +870,7 @@ class TestMembership(FixtureAPITestCase):
"""
Workers may specially have no member as they depends on a repository
"""
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
worker = repo.workers.get(slug='reco')
membership = worker.memberships.create(user=self.user, level=Role.Admin.value)
self.client.force_login(self.user)
@@ -951,7 +951,7 @@ class TestMembership(FixtureAPITestCase):
"""
A worker member can inherit the right to add a member from its repository to edit a worker member
"""
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
worker = repo.workers.get(slug='reco')
repo.memberships.create(user=self.user, level=Role.Admin.value)
@@ -999,7 +999,7 @@ class TestMembership(FixtureAPITestCase):
"""
Workers may have no member as they depends on a repository
"""
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
worker = repo.workers.get(slug='reco')
membership = worker.memberships.create(user=self.user, level=Role.Admin.value)
self.client.force_login(self.user)
@@ -1051,7 +1051,7 @@ class TestMembership(FixtureAPITestCase):
"""
A worker member can inherit the right to add a member from its repository to remove a worker member
"""
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
worker = repo.workers.get(slug='reco')
repo.memberships.create(user=self.user, level=Role.Admin.value)
@@ -1066,13 +1066,13 @@ class TestMembership(FixtureAPITestCase):
new_member.refresh_from_db()
def test_right_unique_user(self):
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
repo.memberships.create(user=self.user, level=Role.Admin.value)
with self.assertRaises(IntegrityError):
repo.memberships.create(user=self.user, level=Role.Admin.value)
def test_right_unique_group(self):
- repo = Repository.objects.get(hook_token='worker-hook-token')
+ repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
repo.memberships.create(group=self.group, level=Role.Admin.value)
with self.assertRaises(IntegrityError):
repo.memberships.create(group=self.group, level=Role.Admin.value)
diff --git a/arkindex/users/tests/test_gitlab_oauth.py b/arkindex/users/tests/test_gitlab_oauth.py
deleted file mode 100644
index a47f6ca89eb6d141dd42dfb878efa916b06414bc..0000000000000000000000000000000000000000
--- a/arkindex/users/tests/test_gitlab_oauth.py
+++ /dev/null
@@ -1,258 +0,0 @@
-import urllib.parse
-from unittest.mock import MagicMock, patch
-
-import responses
-from django.http.request import HttpRequest
-from django.test import override_settings
-from responses import matchers
-from rest_framework.exceptions import AuthenticationFailed
-
-from arkindex.project.tests import FixtureTestCase
-from arkindex.users.models import OAuthStatus
-from arkindex.users.providers import GitLabOAuthProvider
-
-
-class TestGitLabOAuthProvider(FixtureTestCase):
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.creds = cls.user.credentials.get()
- cls.gl_patch = patch('arkindex.users.providers.Gitlab')
-
- def setUp(self):
- super().setUp()
- self.gl_mock = self.gl_patch.start()
-
- def tearDown(self):
- self.gl_patch.stop()
- super().tearDown()
-
- def test_enabled(self):
- with self.settings(GITLAB_APP_ID=None, GITLAB_APP_SECRET=None):
- self.assertFalse(GitLabOAuthProvider.enabled())
- with self.settings(GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='1234'):
- self.assertTrue(GitLabOAuthProvider.enabled())
-
- @override_settings(BACKEND_PUBLIC_URL_OAUTH='http://arkindex.localhost:8000', PUBLIC_HOSTNAME='http://arkindex.localhost:8080')
- def test_callback_uri_dev_override(self):
- self.assertEqual(GitLabOAuthProvider().get_callback_uri(), 'http://arkindex.localhost:8000/api/v1/oauth/providers/gitlab/callback/')
-
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost/')
- def test_callback_uri_public_hostname(self):
- self.assertEqual(GitLabOAuthProvider().get_callback_uri(), 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/')
-
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME=None)
- def test_callback_uri_request(self):
- with self.assertRaisesMessage(AssertionError, 'PUBLIC_HOSTNAME is required to generate callback URIs'):
- GitLabOAuthProvider().get_callback_uri()
-
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost/')
- def test_authorize_uri(self):
- with self.settings(GITLAB_APP_ID='abcd'):
- uri = GitLabOAuthProvider(credentials=self.creds).get_authorize_uri()
- parsed = urllib.parse.urlparse(uri)
- self.assertEqual(parsed.scheme, 'https')
- self.assertEqual(parsed.netloc, 'somewhere')
- self.assertEqual(parsed.path, '/oauth/authorize')
- query = urllib.parse.parse_qs(parsed.query)
- self.assertDictEqual(query, {
- 'client_id': ['abcd'],
- 'redirect_uri': ['https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/'],
- 'scope': ['api'],
- 'response_type': ['code'],
- 'state': [str(self.creds.id)],
- })
-
- def test_handle_callback_no_state(self):
- request_mock = MagicMock(spec=HttpRequest)
- request_mock.user = self.user
- request_mock.GET = {}
- with self.assertRaisesMessage(ValueError, 'No state hash'):
- GitLabOAuthProvider().handle_callback(request_mock)
-
- def test_handle_callback_bad_response(self):
- request_mock = MagicMock(spec=HttpRequest)
- request_mock.user = self.user
- request_mock.GET = {'state': str(self.creds.id)}
- with self.assertRaisesRegex(ValueError, 'valid response'):
- GitLabOAuthProvider().handle_callback(request_mock)
-
- def test_handle_callback_error(self):
- request_mock = MagicMock(spec=HttpRequest)
- request_mock.user = self.user
- request_mock.GET = {'state': str(self.creds.id), 'error': 'error message'}
- with self.assertRaisesMessage(ValueError, 'error message'):
- GitLabOAuthProvider().handle_callback(request_mock)
-
- def test_handle_callback_overwrite(self):
- self.creds.status = OAuthStatus.Done
- self.creds.save()
- request_mock = MagicMock(spec=HttpRequest)
- request_mock.user = self.user
- request_mock.GET = {'state': str(self.creds.id), 'code': 'something'}
- with self.assertRaisesRegex(Exception, 'overwrite'):
- GitLabOAuthProvider().handle_callback(request_mock)
-
- @responses.activate
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_handle_callback_success(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'code': 'abc123',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'authorization_code',
- }),
- ],
- json={
- 'access_token': 't0k3n',
- 'refresh_token': 'r3fr3sh',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
- request_mock = MagicMock(spec=HttpRequest)
- request_mock.user = self.user
- request_mock.GET = {'state': str(self.creds.id), 'code': 'abc123'}
- request_mock.build_absolute_uri.return_value = 'callback'
-
- self.gl_mock.return_value.user.username = 'bobby'
-
- with self.settings(GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t'):
- GitLabOAuthProvider().handle_callback(request_mock)
-
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.token, 't0k3n')
- self.assertEqual(self.creds.refresh_token, 'r3fr3sh')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
- self.assertEqual(self.creds.account_name, 'bobby')
- self.assertEqual(self.creds.status, OAuthStatus.Done)
- self.assertEqual(self.gl_mock.return_value.auth.call_count, 1)
-
- args, kwargs = self.gl_mock.call_args
- self.assertEqual(args, ('https://somewhere', ))
- self.assertDictEqual(kwargs, {'oauth_token': 't0k3n'})
-
- @responses.activate
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_refresh_token(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'r3fr3sh'
- }),
- ],
- json={
- 'access_token': 't0k3n',
- 'refresh_token': 'r4fr4sh',
- 'token_type': 'Bearer',
- 'created_at': 1582984800,
- 'expires_in': '3600',
- },
- )
-
- self.gl_mock.return_value.user.username = 'bobby'
-
- self.creds.refresh_token = 'r3fr3sh'
- self.creds.save()
- self.assertEqual(self.creds.expiry.isoformat(), '2100-12-31T23:59:59.999000+00:00')
- self.assertEqual(self.creds.token, 'oauth-token')
-
- with self.settings(GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t'):
- GitLabOAuthProvider(credentials=self.creds).refresh_token()
-
- self.assertEqual(self.creds.token, 't0k3n')
- self.assertEqual(self.creds.refresh_token, 'r4fr4sh')
- self.assertEqual(self.creds.expiry.isoformat(), '2020-02-29T15:00:00+00:00')
- self.assertEqual(self.creds.account_name, 'bobby')
- self.assertEqual(self.creds.status, OAuthStatus.Done)
- self.assertEqual(self.gl_mock.return_value.auth.call_count, 1)
-
- args, kwargs = self.gl_mock.call_args
- self.assertEqual(args, ('https://somewhere', ))
- self.assertDictEqual(kwargs, {'oauth_token': 't0k3n'})
-
- @responses.activate
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_refresh_token_error(self):
- responses.post(
- 'https://somewhere/oauth/token',
- match=[
- matchers.urlencoded_params_matcher({
- 'client_id': 'abcd',
- 'client_secret': 's3kr3t',
- 'redirect_uri': 'https://arkindex.localhost/api/v1/oauth/providers/gitlab/callback/',
- 'grant_type': 'refresh_token',
- 'refresh_token': 'r3fr3sh'
- }),
- ],
- status=418,
- )
-
- self.creds.refresh_token = 'r3fr3sh'
- self.creds.save()
- self.assertEqual(self.creds.expiry.isoformat(), '2100-12-31T23:59:59.999000+00:00')
- self.assertEqual(self.creds.token, 'oauth-token')
-
- with self.settings(GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t'), self.assertRaises(AuthenticationFailed):
- GitLabOAuthProvider(credentials=self.creds).refresh_token()
-
- self.assertEqual(self.creds.token, 'oauth-token')
- self.assertEqual(self.creds.refresh_token, 'r3fr3sh')
- self.assertEqual(self.creds.expiry.isoformat(), '2100-12-31T23:59:59.999000+00:00')
- self.assertEqual(self.creds.status, OAuthStatus.Error)
- self.assertFalse(self.gl_mock.return_value.auth.called)
-
- @responses.activate
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_refresh_token_none(self):
- """
- GitLabOAuthProvider.refresh_token should not try to refresh the OAuth token if there is
- no refresh token on the OAuthCredentials
- """
- self.creds.refresh_token = None
- self.creds.save()
- self.assertEqual(self.creds.expiry.isoformat(), '2100-12-31T23:59:59.999000+00:00')
- self.assertEqual(self.creds.token, 'oauth-token')
-
- with self.settings(GITLAB_APP_ID='abcd', GITLAB_APP_SECRET='s3kr3t'), self.assertRaises(AuthenticationFailed):
- GitLabOAuthProvider(credentials=self.creds).refresh_token()
-
- self.assertEqual(self.creds.token, 'oauth-token')
- self.assertIsNone(self.creds.refresh_token)
- self.assertEqual(self.creds.expiry.isoformat(), '2100-12-31T23:59:59.999000+00:00')
- self.assertEqual(self.creds.status, OAuthStatus.Error)
- self.assertFalse(self.gl_mock.return_value.auth.called)
-
- @override_settings(BACKEND_PUBLIC_URL_OAUTH=None, PUBLIC_HOSTNAME='https://arkindex.localhost')
- def test_disconnect(self):
- repo_1, repo_2 = self.creds.repos.order_by('hook_token')
- # This hook does not match the expected hook URL and should not be removed
- ignored_hook = MagicMock()
- ignored_hook.url = 'https://potato.localhost/something'
- repo_1_hook = MagicMock()
- repo_1_hook.url = f'https://arkindex.localhost/api/v1/imports/hook/{repo_1.id}/'
- repo_2_hook = MagicMock()
- repo_2_hook.url = f'https://arkindex.localhost/api/v1/imports/hook/{repo_2.id}/'
- self.gl_mock().projects.get.return_value.id = 'repo_id'
- self.gl_mock().projects.get.return_value.hooks.list.return_value = [
- ignored_hook, repo_1_hook, repo_2_hook
- ]
-
- GitLabOAuthProvider(credentials=self.creds).disconnect()
-
- self.assertEqual(self.gl_mock().projects.get.call_count, 2)
- self.assertEqual(self.gl_mock().projects.get.return_value.hooks.list.call_count, 2)
- self.assertFalse(ignored_hook.delete.called)
- self.assertEqual(repo_1_hook.delete.call_count, 1)
- self.assertEqual(repo_2_hook.delete.call_count, 1)
diff --git a/arkindex/users/tests/test_providers.py b/arkindex/users/tests/test_providers.py
deleted file mode 100644
index 3179783fb8913e513e06d113552738e698e56d0d..0000000000000000000000000000000000000000
--- a/arkindex/users/tests/test_providers.py
+++ /dev/null
@@ -1,85 +0,0 @@
-
-from unittest.mock import MagicMock, patch
-
-from django.urls import reverse
-from rest_framework import status
-
-from arkindex.project.tests import FixtureTestCase
-from arkindex.users.models import OAuthStatus
-from arkindex.users.providers import get_provider
-
-
-class TestProviders(FixtureTestCase):
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.creds = cls.user.credentials.get()
-
- @classmethod
- def setUpClass(cls):
- super().setUpClass()
- cls.provider_mock = MagicMock()
- cls.provider_mock.slug = 'provider-slug'
- cls.provider_mock.display_name = 'Provider Name'
- cls.provider_mock.default_url = 'http://somewhere.com'
- cls.provider_mock().credentials = cls.creds
- # Patch global variable providers.oauth_providers
- cls.providers_patch = patch('arkindex.users.providers.oauth_providers', [cls.provider_mock])
-
- def setUp(self):
- super().setUp()
- self.providers_patch.start()
-
- def tearDown(self):
- super().tearDown()
- self.providers_patch.stop()
-
- @classmethod
- def tearDownClass(cls):
- super().tearDownClass()
-
- def test_get_provider(self):
- self.assertEqual(get_provider('provider-slug'), self.provider_mock)
- self.assertIsNone(get_provider('doesnotexist'))
-
- def test_list_providers(self):
- self.client.force_login(self.user)
- response = self.client.get(reverse('api:providers-list'))
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- data = response.json()
- self.assertEqual(data, [
- {
- "name": self.provider_mock.slug,
- "display_name": self.provider_mock.display_name,
- "default_url": self.provider_mock.default_url
- }
- ])
-
- def test_list_providers_requires_login(self):
- response = self.client.get(reverse('api:providers-list'))
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-
- def test_oauth_callback(self):
- self.provider_mock().handle_callback.side_effect = None
- self.client.force_login(self.user)
- self.creds.status = OAuthStatus.Created
- self.creds.save()
- response = self.client.get(
- reverse('api:oauth-callback', kwargs={'provider': 'provider-slug'}),
- )
- self.assertRedirects(response, reverse('credentials'), fetch_redirect_response=False)
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.status, OAuthStatus.Done)
-
- def test_oauth_callback_error(self):
- self.provider_mock().handle_callback.side_effect = ValueError
- self.client.force_login(self.user)
- self.creds.status = OAuthStatus.Created
- self.creds.save()
- response = self.client.get(
- reverse('api:oauth-callback', kwargs={'provider': 'provider-slug'}),
- )
- self.assertRedirects(response, reverse('credentials'), fetch_redirect_response=False)
- self.creds.refresh_from_db()
- self.assertEqual(self.creds.status, OAuthStatus.Error)
diff --git a/requirements.txt b/requirements.txt
index 5c6c517da482a0a41a4404796c9813c62d2a6acf..b98ed6359f777fa1be59de0cd2db8abadb8640ea 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,6 @@ django-rq==2.8.1
djangorestframework==3.12.4
djangorestframework-simplejwt==5.2.2
drf-spectacular==0.18.2
-python-gitlab==3.15.0
python-memcached==1.59
pytz==2023.3
PyYAML==6.0