diff --git a/.isort.cfg b/.isort.cfg index d532ef433bbd25b92e4b31878863e863b7fb4b55..69e050ee6e71a26dbd8ffed12e239ebc499fdc2f 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,5 +7,5 @@ use_parentheses = True line_length = 120 default_section=FIRSTPARTY -known_first_party = ponos,transkribus +known_first_party = transkribus known_third_party = SolrClient,bleach,boto3,botocore,corsheaders,django,django_admin_hstore_widget,django_rq,drf_spectacular,enumfields,gitlab,psycopg2,requests,responses,rest_framework,rq,setuptools,sqlparse,teklia_toolbox,tenacity,tripoli,yaml diff --git a/arkindex/documents/management/commands/bootstrap.py b/arkindex/documents/management/commands/bootstrap.py index 338a8e88abd8a51a3325de3560ab400a20745be3..938befd8eb8ef59a3f785dcbc9e21a23ee4bba83 100644 --- a/arkindex/documents/management/commands/bootstrap.py +++ b/arkindex/documents/management/commands/bootstrap.py @@ -9,6 +9,7 @@ from django.db.utils import IntegrityError from rest_framework.authtoken.models import Token from arkindex.images.models import ImageServer +from arkindex.ponos.models import Farm from arkindex.process.models import ( Repository, Worker, @@ -18,7 +19,6 @@ from arkindex.process.models import ( WorkerVersionState, ) from arkindex.users.models import User -from ponos.models import Farm # Constants used in architecture project IMAGE_SERVER_ID = 12345 diff --git a/arkindex/documents/management/commands/build_fixtures.py b/arkindex/documents/management/commands/build_fixtures.py index 2ea04ad5c39be8dcf1f8d84c810d5746862573c1..08033b2cb431c8c8e0df4c70836557b0fbe69ca4 100644 --- a/arkindex/documents/management/commands/build_fixtures.py +++ b/arkindex/documents/management/commands/build_fixtures.py @@ -8,6 +8,7 @@ from django.utils import timezone as DjangoTimeZone from arkindex.documents.models import Corpus, Element, MetaData, MetaType from arkindex.images.models import Image, ImageServer +from arkindex.ponos.models import Farm, State from arkindex.process.models import ( Process, ProcessMode, @@ -18,7 +19,6 @@ from arkindex.process.models import ( Workflow, ) from arkindex.users.models import Group, Right, Role, User -from ponos.models import Farm, State def fake_now(): diff --git a/arkindex/documents/management/commands/cleanup.py b/arkindex/documents/management/commands/cleanup.py index 71aabd6878929e66a4b27fb06ca33305b6d3cdca..c255870b3f5a9d585682752029fa68c61be6ac18 100644 --- a/arkindex/documents/management/commands/cleanup.py +++ b/arkindex/documents/management/commands/cleanup.py @@ -11,10 +11,10 @@ from django.utils import timezone from arkindex.documents.models import CorpusExport, CorpusExportState, Element from arkindex.images.models import Image, ImageServer +from arkindex.ponos.models import Artifact, Task, Workflow from arkindex.process.models import DataFile, GitRef, GitRefType, WorkerVersion, WorkerVersionState from arkindex.project.aws import s3 from arkindex.training.models import ModelVersion -from ponos.models import Artifact, Task, Workflow # Ponos artifacts use the path: <workflow uuid>/<task id>/<path> # Before June 2020, artifacts used <workflow uuid>/run_<run id>/<task id>.tar.zst diff --git a/arkindex/documents/tests/commands/test_cleanup.py b/arkindex/documents/tests/commands/test_cleanup.py index 233fe70042c474f6e2c94a3dedc2fe17dfd63f6c..0f5f09dc0b843b9431aba30c2c3cc46ac7986cea 100644 --- a/arkindex/documents/tests/commands/test_cleanup.py +++ b/arkindex/documents/tests/commands/test_cleanup.py @@ -10,10 +10,10 @@ from django.test import override_settings from arkindex.documents.models import CorpusExport, CorpusExportState, Element from arkindex.images.models import Image, ImageServer +from arkindex.ponos.models import Artifact, Farm, Task, Workflow from arkindex.process.models import DataFile, GitRefType, Repository, WorkerVersionState from arkindex.project.tests import FixtureTestCase from arkindex.training.models import Model, ModelVersion -from ponos.models import Artifact, Farm, Task, Workflow @override_settings(AWS_EXPORT_BUCKET='export', PONOS_S3_ARTIFACTS_BUCKET='ponos-artifacts', PONOS_S3_LOGS_BUCKET='ponos-logs', AWS_TRAINING_BUCKET='training') diff --git a/arkindex/documents/tests/tasks/test_export.py b/arkindex/documents/tests/tasks/test_export.py index 8a2b53ddd0d0c4c0a7a6393becafa1d9742b6365..d2813ed1bf81d63c23f963c4c4d18d6a9c0f9718 100644 --- a/arkindex/documents/tests/tasks/test_export.py +++ b/arkindex/documents/tests/tasks/test_export.py @@ -22,9 +22,9 @@ from arkindex.documents.models import ( TranscriptionEntity, ) from arkindex.images.models import Image, ImageServer +from arkindex.ponos.models import Artifact from arkindex.process.models import Repository, WorkerType, WorkerVersion, WorkerVersionState from arkindex.project.tests import FixtureTestCase -from ponos.models import Artifact TABLE_NAMES = { 'export_version', diff --git a/arkindex/ponos/__init__.py b/arkindex/ponos/__init__.py index 071fa5fb3e4b85dd8cdc47bb5ee63de475a7031d..92f814fd0906315cc0bb3410740961843b521a67 100644 --- a/arkindex/ponos/__init__.py +++ b/arkindex/ponos/__init__.py @@ -1,3 +1,3 @@ -from ponos.helpers import get_ponos_run, get_ponos_task_id, is_ponos_task # noqa: F401 +from arkindex.ponos.helpers import get_ponos_run, get_ponos_task_id, is_ponos_task # noqa: F401 __all__ = ["is_ponos_task", "get_ponos_task_id", "get_ponos_run"] diff --git a/arkindex/ponos/admin.py b/arkindex/ponos/admin.py index d39d3f0ff14ae4ca579c52d011f8143a95114ae5..d6b6d3a236ee38279d415ef74098bf133a172a6f 100644 --- a/arkindex/ponos/admin.py +++ b/arkindex/ponos/admin.py @@ -4,18 +4,8 @@ from django.core.exceptions import ValidationError from django.db.models import Max, TextField from enumfields.admin import EnumFieldListFilter -from ponos.keys import gen_nonce -from ponos.models import ( - FINAL_STATES, - GPU, - Agent, - Artifact, - Farm, - Secret, - Task, - Workflow, - encrypt, -) +from arkindex.ponos.keys import gen_nonce +from arkindex.ponos.models import FINAL_STATES, GPU, Agent, Artifact, Farm, Secret, Task, Workflow, encrypt class ArtifactInline(admin.TabularInline): diff --git a/arkindex/ponos/api.py b/arkindex/ponos/api.py index 1696b9d4efb808830ce99c72e04f8f07fbbdd065..84e34f2bcbe5b6cef40c6dfcd249762ca3b29857 100644 --- a/arkindex/ponos/api.py +++ b/arkindex/ponos/api.py @@ -20,15 +20,14 @@ from rest_framework.generics import ( from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView -from rest_framework_simplejwt.views import TokenRefreshView -from ponos.authentication import AgentAuthentication -from ponos.keys import load_private_key -from ponos.models import Agent, Artifact, Farm, Secret, State, Task, Workflow -from ponos.permissions import IsAgent, IsAssignedAgentOrReadOnly -from ponos.recipe import parse_recipe -from ponos.renderers import PublicKeyPEMRenderer -from ponos.serializers import ( +from arkindex.ponos.authentication import AgentAuthentication +from arkindex.ponos.keys import load_private_key +from arkindex.ponos.models import Agent, Artifact, Farm, Secret, State, Task, Workflow +from arkindex.ponos.permissions import IsAgent, IsAssignedAgentOrReadOnly +from arkindex.ponos.recipe import parse_recipe +from arkindex.ponos.renderers import PublicKeyPEMRenderer +from arkindex.ponos.serializers import ( AgentActionsSerializer, AgentCreateSerializer, AgentDetailsSerializer, @@ -43,6 +42,7 @@ from ponos.serializers import ( TaskTinySerializer, WorkflowSerializer, ) +from rest_framework_simplejwt.views import TokenRefreshView class PublicKeyEndpoint(APIView): diff --git a/arkindex/ponos/apps.py b/arkindex/ponos/apps.py index 8f83f089c26820d70aa5f18cf85be81fd77f57b0..c59511a4387bb4f5531c45a1bb9e42db2f492cec 100644 --- a/arkindex/ponos/apps.py +++ b/arkindex/ponos/apps.py @@ -2,5 +2,5 @@ from django.apps import AppConfig class PonosConfig(AppConfig): - name = "ponos" + name = "arkindex.ponos" verbose_name = "Ponos: tasks manager" diff --git a/arkindex/ponos/authentication.py b/arkindex/ponos/authentication.py index 327f2b11809e1d03fe16b069551561bf323735fe..58192cb896542974bc15a4c3d59402de28acd6d4 100644 --- a/arkindex/ponos/authentication.py +++ b/arkindex/ponos/authentication.py @@ -1,11 +1,11 @@ from drf_spectacular.contrib.rest_framework_simplejwt import SimpleJWTScheme from rest_framework.exceptions import AuthenticationFailed + +from arkindex.ponos.models import Agent from rest_framework_simplejwt.authentication import JWTAuthentication from rest_framework_simplejwt.exceptions import InvalidToken from rest_framework_simplejwt.settings import api_settings -from ponos.models import Agent - class AgentUser(Agent): """ diff --git a/arkindex/ponos/fields.py b/arkindex/ponos/fields.py index a7d91ce39427fd6a45d404341ff3f2cc7f6aa1fb..19f690aa5a03b3a6851b428d8fd00ebd7a989ee8 100644 --- a/arkindex/ponos/fields.py +++ b/arkindex/ponos/fields.py @@ -1,7 +1,7 @@ from django.core.exceptions import ValidationError from django.db.models import CharField, JSONField -from ponos import forms +from arkindex.ponos import forms class CommaSeparatedListField(CharField): diff --git a/arkindex/ponos/management/commands/generate_private_key.py b/arkindex/ponos/management/commands/generate_private_key.py index d0ca0ae550612c6551acbf59d00caf1a6f632c1e..9f4161b66bc183e7e75f74f946428121c78e7e60 100644 --- a/arkindex/ponos/management/commands/generate_private_key.py +++ b/arkindex/ponos/management/commands/generate_private_key.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand -from ponos.keys import gen_private_key +from arkindex.ponos.keys import gen_private_key class Command(BaseCommand): diff --git a/arkindex/ponos/migrations/0001_initial.py b/arkindex/ponos/migrations/0001_initial.py index f9af1350750aaa06ccedad4c2e76ca5da4fe885f..083fb7c137f16e0e78805b6fc26612aac5358cbe 100644 --- a/arkindex/ponos/migrations/0001_initial.py +++ b/arkindex/ponos/migrations/0001_initial.py @@ -6,7 +6,7 @@ import django.db.models.deletion import enumfields.fields from django.db import migrations, models -import ponos.models +import arkindex.ponos.models class Migration(migrations.Migration): @@ -31,7 +31,7 @@ class Migration(migrations.Migration): ( "state", enumfields.fields.EnumField( - default="unscheduled", enum=ponos.models.State, max_length=20 + default="unscheduled", enum=arkindex.ponos.models.State, max_length=20 ), ), ("container", models.CharField(blank=True, max_length=64, null=True)), diff --git a/arkindex/ponos/migrations/0002_recipe_validator.py b/arkindex/ponos/migrations/0002_recipe_validator.py index 80d380e4fb003b052d81a38f065bfa6c55614ab9..e03f3ddb8c7f05e8c68615fc55596bb7d8c1c164 100644 --- a/arkindex/ponos/migrations/0002_recipe_validator.py +++ b/arkindex/ponos/migrations/0002_recipe_validator.py @@ -2,7 +2,7 @@ from django.db import migrations, models -import ponos.models +import arkindex.ponos.models class Migration(migrations.Migration): @@ -15,6 +15,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="workflow", name="recipe", - field=models.TextField(validators=[ponos.models.recipe_validator]), + field=models.TextField(validators=[arkindex.ponos.models.recipe_validator]), ), ] diff --git a/arkindex/ponos/migrations/0009_tags.py b/arkindex/ponos/migrations/0009_tags.py index 667b586caad0e5ce3fefad0ab25af4d977574f6e..c608f189f0992e089233455254d69a20a057e9c2 100644 --- a/arkindex/ponos/migrations/0009_tags.py +++ b/arkindex/ponos/migrations/0009_tags.py @@ -2,7 +2,7 @@ from django.db import migrations -import ponos.fields +import arkindex.ponos.fields class Migration(migrations.Migration): @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name="agent", name="tags", - field=ponos.fields.CommaSeparatedListField(default=list, max_length=250), + field=arkindex.ponos.fields.CommaSeparatedListField(default=list, max_length=250), ), migrations.AddField( model_name="task", name="tags", - field=ponos.fields.CommaSeparatedListField(default=list, max_length=250), + field=arkindex.ponos.fields.CommaSeparatedListField(default=list, max_length=250), ), ] diff --git a/arkindex/ponos/migrations/0010_farm.py b/arkindex/ponos/migrations/0010_farm.py index 8ca3cf970ffbb4b100a2e0ab3285b015abb6d276..8f01d0d329e777a2971bea8a9c64ecdc914e19c2 100644 --- a/arkindex/ponos/migrations/0010_farm.py +++ b/arkindex/ponos/migrations/0010_farm.py @@ -6,7 +6,7 @@ import django.core.validators import django.db.models.deletion from django.db import migrations, models -import ponos.models +import arkindex.ponos.models def default_farm(apps, schema_editor): @@ -47,7 +47,7 @@ class Migration(migrations.Migration): ( "seed", models.CharField( - default=ponos.models.generate_seed, + default=arkindex.ponos.models.generate_seed, max_length=64, unique=True, validators=[ diff --git a/arkindex/ponos/migrations/0012_advanced_tags.py b/arkindex/ponos/migrations/0012_advanced_tags.py index 2a98bed6e6f340e9cc39bee0e3d36c0637fb7195..3915e818854613341d8fe64b05a6af78c924a528 100644 --- a/arkindex/ponos/migrations/0012_advanced_tags.py +++ b/arkindex/ponos/migrations/0012_advanced_tags.py @@ -2,7 +2,7 @@ from django.db import migrations -import ponos.fields +import arkindex.ponos.fields class Migration(migrations.Migration): @@ -20,6 +20,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name="agent", name="exclude_tags", - field=ponos.fields.CommaSeparatedListField(default=list, max_length=250), + field=arkindex.ponos.fields.CommaSeparatedListField(default=list, max_length=250), ), ] diff --git a/arkindex/ponos/migrations/0014_modify_task_model.py b/arkindex/ponos/migrations/0014_modify_task_model.py index ed0393f195262473413268753960da2d49aaef29..3e6a4633b62841ec6c4a91f85121e8315f4b1c13 100644 --- a/arkindex/ponos/migrations/0014_modify_task_model.py +++ b/arkindex/ponos/migrations/0014_modify_task_model.py @@ -2,7 +2,7 @@ from django.db import migrations, models -from ponos.recipe import parse_recipe +from arkindex.ponos.recipe import parse_recipe try: import jsonfield diff --git a/arkindex/ponos/migrations/0019_secret.py b/arkindex/ponos/migrations/0019_secret.py index cb30c62056dd16f064601b52e47f992ab2e550cd..bb33c07f76f45428bf23c489f5e4dfffd06404f5 100644 --- a/arkindex/ponos/migrations/0019_secret.py +++ b/arkindex/ponos/migrations/0019_secret.py @@ -4,7 +4,7 @@ import uuid from django.db import migrations, models -import ponos.keys +import arkindex.ponos.keys class Migration(migrations.Migration): @@ -26,7 +26,7 @@ class Migration(migrations.Migration): ("name", models.CharField(max_length=250, unique=True)), ( "nonce", - models.BinaryField(default=ponos.keys.gen_nonce, max_length=16), + models.BinaryField(default=arkindex.ponos.keys.gen_nonce, max_length=16), ), ("content", models.BinaryField(editable=True)), ], diff --git a/arkindex/ponos/migrations/0029_task_expiry.py b/arkindex/ponos/migrations/0029_task_expiry.py index 932463a91f091803220d127d00d743681d1d84a5..662f5f7e32aae84574116f97f19064f9af2f6908 100644 --- a/arkindex/ponos/migrations/0029_task_expiry.py +++ b/arkindex/ponos/migrations/0029_task_expiry.py @@ -4,7 +4,7 @@ from datetime import timedelta from django.db import migrations, models -from ponos.models import expiry_default +from arkindex.ponos.models import expiry_default def set_expiry(apps, schema_editor): diff --git a/arkindex/ponos/migrations/0031_emptyable_jsonfield.py b/arkindex/ponos/migrations/0031_emptyable_jsonfield.py index f1b9bef89df4104d9a0a9d58c88d94de77802c1e..10a7d1721f6746a9e39220fc21c72962e2d11853 100644 --- a/arkindex/ponos/migrations/0031_emptyable_jsonfield.py +++ b/arkindex/ponos/migrations/0031_emptyable_jsonfield.py @@ -2,7 +2,7 @@ from django.db import migrations -import ponos.fields +import arkindex.ponos.fields class Migration(migrations.Migration): @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="task", name="env", - field=ponos.fields.StringDictField(blank=True, null=True), + field=arkindex.ponos.fields.StringDictField(blank=True, null=True), ), migrations.AlterField( model_name="task", name="extra_files", - field=ponos.fields.StringDictField(default=dict), + field=arkindex.ponos.fields.StringDictField(default=dict), ), ] diff --git a/arkindex/ponos/migrations/0034_alter_artifact_size.py b/arkindex/ponos/migrations/0034_alter_artifact_size.py index 06a5d668f549b36cbea97fd43e2986b0f6357835..94a378d25a725b5bbde2bd66f41f25660898147f 100644 --- a/arkindex/ponos/migrations/0034_alter_artifact_size.py +++ b/arkindex/ponos/migrations/0034_alter_artifact_size.py @@ -3,8 +3,8 @@ from django.core.validators import MinValueValidator from django.db import migrations, models -from ponos.models import artifact_max_size -from ponos.validators import MaxValueValidator +from arkindex.ponos.models import artifact_max_size +from arkindex.ponos.validators import MaxValueValidator class Migration(migrations.Migration): diff --git a/arkindex/ponos/models.py b/arkindex/ponos/models.py index 7ef1b2994c752f7d62de75a621b41e268ce02f20..1adee90bbd968cff76fd862d57aa78f731dcbeb9 100644 --- a/arkindex/ponos/models.py +++ b/arkindex/ponos/models.py @@ -7,7 +7,6 @@ from datetime import timedelta from hashlib import sha256 from botocore.exceptions import ClientError -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from django.conf import settings from django.core.exceptions import ValidationError from django.core.validators import MinLengthValidator, MinValueValidator, RegexValidator @@ -17,15 +16,16 @@ from django.urls import reverse from django.utils import timezone from django.utils.functional import cached_property from enumfields import Enum, EnumField -from rest_framework_simplejwt.tokens import RefreshToken from yaml import YAMLError -from ponos.aws import object_url, s3 -from ponos.fields import CommaSeparatedListField, StringDictField -from ponos.keys import gen_nonce -from ponos.managers import TaskManager -from ponos.recipe import parse_recipe, recipe_depth -from ponos.validators import MaxValueValidator +from arkindex.ponos.aws import object_url, s3 +from arkindex.ponos.fields import CommaSeparatedListField, StringDictField +from arkindex.ponos.keys import gen_nonce +from arkindex.ponos.managers import TaskManager +from arkindex.ponos.recipe import parse_recipe, recipe_depth +from arkindex.ponos.validators import MaxValueValidator +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from rest_framework_simplejwt.tokens import RefreshToken # Maximum allowed time until an agent is considered inactive since last request AGENT_TIMEOUT = timedelta( diff --git a/arkindex/ponos/permissions.py b/arkindex/ponos/permissions.py index 69525e6be6bd3370e12f8de2df80ce70d74fbe41..77f1a814b5751d4c0c4a636de63c2cfe01210c9a 100644 --- a/arkindex/ponos/permissions.py +++ b/arkindex/ponos/permissions.py @@ -1,6 +1,6 @@ from rest_framework.permissions import SAFE_METHODS, IsAuthenticated -from ponos.models import Task +from arkindex.ponos.models import Task class IsAgent(IsAuthenticated): diff --git a/arkindex/ponos/serializers.py b/arkindex/ponos/serializers.py index d0b067468770072219d7117562cbb9dbdff2669a..daaefd1a9ab6593ac6b2047a84933cc51ef84b74 100644 --- a/arkindex/ponos/serializers.py +++ b/arkindex/ponos/serializers.py @@ -9,8 +9,8 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers from rest_framework.exceptions import ValidationError -from ponos.keys import check_agent_key -from ponos.models import ( +from arkindex.ponos.keys import check_agent_key +from arkindex.ponos.models import ( ACTIVE_STATES, FINAL_STATES, GPU, @@ -23,8 +23,8 @@ from ponos.models import ( Task, Workflow, ) -from ponos.serializer_fields import Base64Field, EnumField, PublicKeyField -from ponos.signals import task_failure +from arkindex.ponos.serializer_fields import Base64Field, EnumField, PublicKeyField +from arkindex.ponos.signals import task_failure logger = logging.getLogger(__name__) diff --git a/arkindex/ponos/tests/test_admin.py b/arkindex/ponos/tests/test_admin.py index 1a8def09605229cb920d98550bff9d84ab576d46..1e56a5d206e6f1086405387d1af9e1b7f97a592b 100644 --- a/arkindex/ponos/tests/test_admin.py +++ b/arkindex/ponos/tests/test_admin.py @@ -1,7 +1,7 @@ from django.test import TestCase -from ponos.admin import ClearTextSecretForm, SecretAdmin -from ponos.models import Secret, encrypt +from arkindex.ponos.admin import ClearTextSecretForm, SecretAdmin +from arkindex.ponos.models import Secret, encrypt class TestAdmin(TestCase): diff --git a/arkindex/ponos/tests/test_api.py b/arkindex/ponos/tests/test_api.py index 5e8c0a3472522e191f0f40710172dda2a0243241..62cad0d07b7091b7483ee7a85b8ecc99794b725f 100644 --- a/arkindex/ponos/tests/test_api.py +++ b/arkindex/ponos/tests/test_api.py @@ -5,11 +5,6 @@ import uuid from io import BytesIO from unittest.mock import call, patch -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.hashes import SHA256 -from cryptography.hazmat.primitives.kdf.hkdf import HKDF -from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from django.conf import settings from django.contrib.auth.models import User from django.test import override_settings @@ -18,19 +13,14 @@ from django.utils import timezone from rest_framework import status from rest_framework.test import APITestCase -from ponos.api import timezone as api_tz -from ponos.authentication import AgentUser -from ponos.models import ( - FINAL_STATES, - GPU, - Agent, - Farm, - Secret, - State, - Task, - Workflow, - encrypt, -) +from arkindex.ponos.api import timezone as api_tz +from arkindex.ponos.authentication import AgentUser +from arkindex.ponos.models import FINAL_STATES, GPU, Agent, Farm, Secret, State, Task, Workflow, encrypt +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.hashes import SHA256 +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from tests.helpers import build_public_key RECIPE = """ diff --git a/arkindex/ponos/tests/test_keys.py b/arkindex/ponos/tests/test_keys.py index d25cb5a9420cf51311d0f8a891791f886b46b31c..697c5717d6f7c576bb78355a22e31b855740d96a 100644 --- a/arkindex/ponos/tests/test_keys.py +++ b/arkindex/ponos/tests/test_keys.py @@ -1,10 +1,10 @@ import os import tempfile -import cryptography from django.test import TestCase, override_settings -from ponos.keys import gen_private_key, load_private_key +import cryptography +from arkindex.ponos.keys import gen_private_key, load_private_key BAD_KEY = """-----BEGIN RSA PRIVATE KEY----- MCoCAQACBGvoyx0CAwEAAQIEUg2X0QIDAMUbAgMAjCcCAmr9AgJMawICKvo= diff --git a/arkindex/ponos/tests/test_models.py b/arkindex/ponos/tests/test_models.py index 8c3e20e85fbcdbf32734230a27ce901a8d866831..d7ba8ae4dcfd3b1a904e60223f8666d0f0ccc46b 100644 --- a/arkindex/ponos/tests/test_models.py +++ b/arkindex/ponos/tests/test_models.py @@ -6,16 +6,7 @@ from django.db.models import prefetch_related_objects from django.test import TestCase, override_settings from django.utils import timezone -from ponos.models import ( - FINAL_STATES, - Agent, - Farm, - Secret, - State, - Workflow, - build_aes_cipher, - encrypt, -) +from arkindex.ponos.models import FINAL_STATES, Agent, Farm, Secret, State, Workflow, build_aes_cipher, encrypt RECIPE = """ tasks: diff --git a/arkindex/ponos/tests/test_recipe.py b/arkindex/ponos/tests/test_recipe.py index 18ef2dbe9aaf879659cb2e4e71164b9248d30a45..78c2aea5b90f0df756f154036d51162900fb1ea1 100644 --- a/arkindex/ponos/tests/test_recipe.py +++ b/arkindex/ponos/tests/test_recipe.py @@ -2,7 +2,7 @@ from textwrap import dedent from django.test import TestCase -from ponos.recipe import parse_recipe +from arkindex.ponos.recipe import parse_recipe # List of (broken recipe, expected AssertionError message) tuples ERROR_CASES = [ diff --git a/arkindex/ponos/tests/test_tasks_attribution.py b/arkindex/ponos/tests/test_tasks_attribution.py index 84738085852ef00087c23e77b067cc1ca45b6f07..52c2117768fb558b5f414f1450e532e1b0d395ec 100644 --- a/arkindex/ponos/tests/test_tasks_attribution.py +++ b/arkindex/ponos/tests/test_tasks_attribution.py @@ -5,8 +5,8 @@ from unittest.mock import patch from django.test import TestCase from django.utils import timezone -from ponos.models import Agent, Farm, State, Task, Workflow -from ponos.models import timezone as model_tz +from arkindex.ponos.models import Agent, Farm, State, Task, Workflow +from arkindex.ponos.models import timezone as model_tz class TasksAttributionTestCase(TestCase): diff --git a/arkindex/ponos/tests/test_util.py b/arkindex/ponos/tests/test_util.py index 634aaa746d91ee06d1a1fd62f9c5804590891821..a85f8c538189e99c3f0d1c561a1c542213223764 100644 --- a/arkindex/ponos/tests/test_util.py +++ b/arkindex/ponos/tests/test_util.py @@ -2,7 +2,7 @@ import uuid from unittest import TestCase from unittest.mock import patch -from ponos import get_ponos_run, get_ponos_task_id, is_ponos_task +from arkindex.ponos import get_ponos_run, get_ponos_task_id, is_ponos_task class TestUtil(TestCase): diff --git a/arkindex/ponos/tests/test_workflow.py b/arkindex/ponos/tests/test_workflow.py index c6dd0a5642e9f76180a70c3c60ef169dfaf1449d..b4e8ea1de72aad35769df01137c501c73e87c6fc 100644 --- a/arkindex/ponos/tests/test_workflow.py +++ b/arkindex/ponos/tests/test_workflow.py @@ -5,7 +5,7 @@ from django.urls import reverse from django.utils import timezone from rest_framework import status -from ponos.models import Agent, Farm, State, Workflow +from arkindex.ponos.models import Agent, Farm, State, Workflow ONE_TASK = """ tasks: diff --git a/arkindex/ponos/urls.py b/arkindex/ponos/urls.py index 7917b0dbb27fddfd08fd79b5a7eef1a288e1238d..67853c54d54602dd2a325c7edbc68a87a1d79d9d 100644 --- a/arkindex/ponos/urls.py +++ b/arkindex/ponos/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from ponos.api import ( +from arkindex.ponos.api import ( AgentActions, AgentDetails, AgentRegister, diff --git a/arkindex/process/api.py b/arkindex/process/api.py index 6f9cfc3cae73bcd4381a8b78aef332165b7e684a..b8252b1ec057342d6daca555eb08fb926a3aeeb8 100644 --- a/arkindex/process/api.py +++ b/arkindex/process/api.py @@ -38,6 +38,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from arkindex.documents.models import Corpus, Element +from arkindex.ponos.models import STATES_ORDERING, State from arkindex.process.models import ( ActivityState, DataFile, @@ -104,7 +105,6 @@ from arkindex.project.tools import PercentileCont, RTrimChr from arkindex.project.triggers import process_delete from arkindex.users.models import OAuthCredentials, Role, Scope from arkindex.users.utils import get_max_level -from ponos.models import STATES_ORDERING, State logger = logging.getLogger(__name__) diff --git a/arkindex/process/models.py b/arkindex/process/models.py index e95912170858bcb08c23721e84be2f57847d02d0..bbdf445ec9efc794eeb2c48c2c7f3573f74a9afa 100644 --- a/arkindex/process/models.py +++ b/arkindex/process/models.py @@ -17,13 +17,13 @@ from rest_framework.exceptions import ValidationError import pgtrigger from arkindex.documents.models import Element from arkindex.images.models import ImageServer +from arkindex.ponos.models import Artifact, State, Workflow from arkindex.process.managers import ActivityManager, CorpusWorkerVersionManager from arkindex.process.providers import get_provider from arkindex.process.utils import get_default_farm_id from arkindex.project.aws import S3FileMixin, S3FileStatus from arkindex.project.fields import ArrayField, MD5HashField from arkindex.project.models import IndexableModel -from ponos.models import Artifact, State, Workflow class ActivityState(Enum): diff --git a/arkindex/process/serializers/imports.py b/arkindex/process/serializers/imports.py index 552942a1fe7b820eba210c8c309df60a598a9807..56514247f3c0ab32bb64fdf5f7564a10acdc8fa4 100644 --- a/arkindex/process/serializers/imports.py +++ b/arkindex/process/serializers/imports.py @@ -4,13 +4,13 @@ from rest_framework import serializers from rest_framework.exceptions import PermissionDenied, ValidationError from arkindex.documents.models import Corpus, Element, ElementType +from arkindex.ponos.models import Farm, State from arkindex.process.models import ActivityState, DataFile, Process, ProcessMode, WorkerRun, WorkerVersionState from arkindex.process.serializers.git import RevisionSerializer from arkindex.project.mixins import ProcessACLMixin from arkindex.project.serializer_fields import EnumField, LinearRingField from arkindex.users.models import Role from arkindex.users.utils import get_max_level -from ponos.models import Farm, State from transkribus import TranskribusAPI diff --git a/arkindex/process/signals.py b/arkindex/process/signals.py index b4a96c94ee92a6543d4c5bbe1d1cde79d51a39ed..ad23523dcb88f190f170d85bbf13454b630459d3 100644 --- a/arkindex/process/signals.py +++ b/arkindex/process/signals.py @@ -4,9 +4,9 @@ from django.db.models.signals import pre_save from django.dispatch import receiver from rest_framework.exceptions import ValidationError +from arkindex.ponos.signals import task_failure from arkindex.process.models import ActivityState, Process, WorkerActivityState, WorkerConfiguration, WorkerRun from arkindex.process.utils import hash_object -from ponos.signals import task_failure logger = logging.getLogger(__name__) diff --git a/arkindex/process/tests/test_create_training_process.py b/arkindex/process/tests/test_create_training_process.py index 7cd77bfde23677873ce8a1d58110da9591e4da5e..0ea843f556185b04f86267ab3469af05b6071f7f 100644 --- a/arkindex/process/tests/test_create_training_process.py +++ b/arkindex/process/tests/test_create_training_process.py @@ -6,6 +6,8 @@ from django.test import override_settings from django.urls import reverse from rest_framework import status +from arkindex.ponos.models import Artifact +from arkindex.ponos.models import State as PonosState from arkindex.process.models import ( Process, ProcessMode, @@ -18,8 +20,6 @@ from arkindex.process.models import ( from arkindex.project.tests import FixtureTestCase from arkindex.training.models import Model, ModelVersion, ModelVersionState from arkindex.users.models import Role -from ponos.models import Artifact -from ponos.models import State as PonosState class TestCreateTrainingProcess(FixtureTestCase): diff --git a/arkindex/process/tests/test_managers.py b/arkindex/process/tests/test_managers.py index 9b2e3520a5b264af1d735c3aef9e6d74900e5d64..063f62f27eb9eacf15c27f47117861a8bcddc605 100644 --- a/arkindex/process/tests/test_managers.py +++ b/arkindex/process/tests/test_managers.py @@ -1,9 +1,9 @@ from uuid import uuid4 from arkindex.documents.models import Classification, Element, Entity, MetaData, Transcription, TranscriptionEntity +from arkindex.ponos.models import Artifact from arkindex.process.models import CorpusWorkerVersion, Repository, WorkerType, WorkerVersionState from arkindex.project.tests import FixtureTestCase -from ponos.models import Artifact class TestManagers(FixtureTestCase): diff --git a/arkindex/process/tests/test_processes.py b/arkindex/process/tests/test_processes.py index 1dce5502ee6ae27288e08011b8163107890813c8..e443de37d1914151eedfb4e5eac0f7699e01a928 100644 --- a/arkindex/process/tests/test_processes.py +++ b/arkindex/process/tests/test_processes.py @@ -11,6 +11,7 @@ from rest_framework import status from rest_framework.exceptions import ValidationError from arkindex.documents.models import Corpus, ElementType +from arkindex.ponos.models import Farm, State, Task, Workflow from arkindex.process.models import ( ActivityState, Process, @@ -24,7 +25,6 @@ from arkindex.process.utils import get_default_farm_id from arkindex.project.tests import FixtureAPITestCase from arkindex.training.models import Model, ModelVersion, ModelVersionState from arkindex.users.models import Role, User -from ponos.models import Farm, State, Task, Workflow RECIPE = ''' tasks: diff --git a/arkindex/process/tests/test_repos.py b/arkindex/process/tests/test_repos.py index 4fcdfeba8cd443a6f3466cf6bca7f71f6f4515d9..d197d01995fa6e03bf8df930df7b3faa87791747 100644 --- a/arkindex/process/tests/test_repos.py +++ b/arkindex/process/tests/test_repos.py @@ -5,10 +5,10 @@ from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.serializers import DateTimeField +from arkindex.ponos.models import State, Workflow from arkindex.process.models import ActivityState, Process, ProcessMode, Repository from arkindex.project.tests import FixtureTestCase from arkindex.users.models import Role, User -from ponos.models import State, Workflow RECIPE = ''' tasks: diff --git a/arkindex/process/tests/test_signals.py b/arkindex/process/tests/test_signals.py index 645fed865d5edd56a6d71f88b0176e955892a0c0..ae384d45b31c5680904a7a78c647fe470742b957 100644 --- a/arkindex/process/tests/test_signals.py +++ b/arkindex/process/tests/test_signals.py @@ -4,6 +4,7 @@ from django.urls import reverse from rest_framework import status from rest_framework.exceptions import ValidationError +from arkindex.ponos.models import State from arkindex.process.models import ( ActivityState, ProcessMode, @@ -15,7 +16,6 @@ from arkindex.process.models import ( ) from arkindex.process.signals import _list_ancestors from arkindex.project.tests import FixtureAPITestCase -from ponos.models import State class TestSignals(FixtureAPITestCase): diff --git a/arkindex/process/tests/test_transkribus_import.py b/arkindex/process/tests/test_transkribus_import.py index 0da526005b8604fd73cacb3986ca5fe5db5e7668..b1c032ddb8d2131e0ab6dbba1112beafc009adf9 100644 --- a/arkindex/process/tests/test_transkribus_import.py +++ b/arkindex/process/tests/test_transkribus_import.py @@ -6,11 +6,11 @@ from django.test import override_settings from django.urls import reverse from rest_framework import status +from arkindex.ponos.models import State from arkindex.process.models import Process, ProcessMode, Repository, Revision, Worker, WorkerType, WorkerVersion from arkindex.project.default_corpus import DEFAULT_TRANSKRIBUS_TYPES from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role, User -from ponos.models import State @override_settings(TRANSKRIBUS_EMAIL="arkindex@teklia.com", TRANSKRIBUS_PASSWORD=None) diff --git a/arkindex/process/tests/test_utils.py b/arkindex/process/tests/test_utils.py index 65c3314bc389e4f612435dc1325a3a9ebb76e7e9..baa86178c4c7155e4d60b592c18bf72ecab4454a 100644 --- a/arkindex/process/tests/test_utils.py +++ b/arkindex/process/tests/test_utils.py @@ -2,8 +2,8 @@ import uuid from django.test import TestCase, override_settings +from arkindex.ponos.models import Farm from arkindex.process.utils import get_default_farm_id -from ponos.models import Farm DEFAULT_FARM_ID = str(uuid.uuid4()) diff --git a/arkindex/process/tests/test_workerruns.py b/arkindex/process/tests/test_workerruns.py index 510c33f551cc7918fcb835a241abc1fc2881bd31..24a27695bc1eca74f4e0c4bb9ea47e04569990ed 100644 --- a/arkindex/process/tests/test_workerruns.py +++ b/arkindex/process/tests/test_workerruns.py @@ -5,12 +5,12 @@ from django.test import override_settings from django.urls import reverse from rest_framework import status +from arkindex.ponos.models import State, Workflow from arkindex.process.models import GitRefType, ProcessMode, Revision, WorkerRun, WorkerVersion, WorkerVersionState from arkindex.process.utils import get_default_farm_id from arkindex.project.tests import FixtureAPITestCase from arkindex.training.models import Model, ModelVersion, ModelVersionState from arkindex.users.models import Role -from ponos.models import State, Workflow RECIPE = ''' tasks: diff --git a/arkindex/process/tests/test_workers.py b/arkindex/process/tests/test_workers.py index f34c9cb83b98cd3b37ce491962665813b2cb03f8..dd570883336510ec5d1afde8ce6a918df5e85103 100644 --- a/arkindex/process/tests/test_workers.py +++ b/arkindex/process/tests/test_workers.py @@ -3,6 +3,7 @@ import uuid from django.urls import reverse from rest_framework import status +from arkindex.ponos.models import Workflow from arkindex.process.models import ( GitRefType, Repository, @@ -16,7 +17,6 @@ from arkindex.process.models import ( from arkindex.process.utils import get_default_farm_id from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Right, Role, User -from ponos.models import Workflow RECIPE = ''' tasks: diff --git a/arkindex/process/tests/test_workflows_api.py b/arkindex/process/tests/test_workflows_api.py index afa5375bc580982c27a2cf6ee0f40c309eb9885b..9fdc0ea3a7fedcabbfd87c02684477f6a7a8e98f 100644 --- a/arkindex/process/tests/test_workflows_api.py +++ b/arkindex/process/tests/test_workflows_api.py @@ -7,12 +7,12 @@ from rest_framework import status from rest_framework.reverse import reverse from arkindex.documents.models import Corpus, Element +from arkindex.ponos.models import State, Workflow from arkindex.process.models import ActivityState, Process, ProcessMode, WorkerActivity, WorkerVersion from arkindex.process.utils import get_default_farm_id from arkindex.project.mixins import SelectionMixin from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role -from ponos.models import State, Workflow RECIPE = ''' tasks: diff --git a/arkindex/project/checks.py b/arkindex/project/checks.py index 55cbbe98080155476e13dad7142c32c1da47c3ee..7a61e4541c99dc225f69290f5feb3b057f5e9ef9 100644 --- a/arkindex/project/checks.py +++ b/arkindex/project/checks.py @@ -11,8 +11,8 @@ import sys import yaml from django.core.checks import Critical, Error, Warning, register +from arkindex.ponos.recipe import parse_recipe from arkindex.process.models import WorkerVersion -from ponos.recipe import parse_recipe def only_runserver(func): diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index eea424435a3e4037ca3c6321d4b561f0b03203c6..5d5bee89890add5949ce80194ccc0e233cc8a5f2 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -110,9 +110,9 @@ INSTALLED_APPS = [ 'django_rq', 'drf_spectacular', 'pgtrigger', - 'ponos', # Our apps + 'arkindex.ponos', 'arkindex.images', 'arkindex.documents', 'arkindex.users', @@ -201,7 +201,7 @@ REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ( 'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.TokenAuthentication', - 'ponos.authentication.AgentAuthentication', + 'arkindex.ponos.authentication.AgentAuthentication', ), 'DEFAULT_PAGINATION_CLASS': 'arkindex.project.pagination.PageNumberPagination', 'DEFAULT_SCHEMA_CLASS': 'arkindex.project.openapi.AutoSchema', @@ -245,7 +245,7 @@ SPECTACULAR_SETTINGS = { 'EntityType': 'arkindex.documents.models.EntityType', 'S3FileStatus': 'arkindex.project.aws.S3FileStatus', 'ClassificationState': 'arkindex.documents.models.ClassificationState', - 'PonosState': 'ponos.models.State', + 'PonosState': 'arkindex.ponos.models.State', 'WorkerVersionState': 'arkindex.process.models.WorkerVersionState', 'ModelVersionState': 'arkindex.training.models.ModelVersionState', 'MetricMode': 'arkindex.training.models.MetricMode', diff --git a/arkindex/project/tests/test_ponos_view.py b/arkindex/project/tests/test_ponos_view.py index 572a5d6ba5f278962b7029b0403d371e09008f90..cabb9626d3d497920b22d3e73c25f15f02a6b185 100644 --- a/arkindex/project/tests/test_ponos_view.py +++ b/arkindex/project/tests/test_ponos_view.py @@ -6,11 +6,11 @@ from django.urls import reverse from rest_framework import status from arkindex.documents.models import Corpus +from arkindex.ponos.authentication import AgentUser +from arkindex.ponos.models import Agent, Artifact, Farm, Secret, State, encrypt from arkindex.process.models import Process, ProcessMode from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role, User -from ponos.authentication import AgentUser -from ponos.models import Agent, Artifact, Farm, Secret, State, encrypt @override_settings(PONOS_PRIVATE_KEY='staging') diff --git a/arkindex/project/views.py b/arkindex/project/views.py index 116ce1b4e669ca9b15d9fc92b4ffd91eb015b9e6..b2c386bad6c75da220f3ff09aebfe07958238603 100644 --- a/arkindex/project/views.py +++ b/arkindex/project/views.py @@ -5,12 +5,19 @@ from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import permissions from rest_framework.exceptions import NotFound, PermissionDenied +from arkindex.ponos.api import ( + AgentDetails, + AgentsState, + SecretDetails, + TaskArtifactDownload, + TaskUpdate, + WorkflowDetails, +) from arkindex.process.models import Process from arkindex.process.permissions import IsArtifactAdmin, IsTaskAdmin from arkindex.project.mixins import CachedViewMixin, ProcessACLMixin from arkindex.project.permissions import IsInternal, IsVerified from arkindex.users.models import Role -from ponos.api import AgentDetails, AgentsState, SecretDetails, TaskArtifactDownload, TaskUpdate, WorkflowDetails class FrontendView(View):