diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 674ed23fd646d34c1df21137bfebbfd47997f8bb..1afa534c0ba947c67b6db153957ce1aab7f7b7a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,6 @@ include: before_script: # Custom line to install our own deps from Git using GitLab CI credentials - - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/common#egg=arkindex-common" - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/ponos#egg=ponos-server" - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/transkribus#egg=transkribus-client" - pip install -r tests-requirements.txt codecov diff --git a/.isort.cfg b/.isort.cfg index b07c8654e194642f00a0bb862f84fda86f09255a..6bae546cc08cd6c91786cf5cc219c6deae143407 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,5 +7,5 @@ use_parentheses = True line_length = 120 default_section=FIRSTPARTY -known_first_party = arkindex_common,ponos,transkribus +known_first_party = ponos,transkribus known_third_party = boto3,botocore,corsheaders,django,django_admin_hstore_widget,django_rq,drf_spectacular,elasticsearch,elasticsearch_dsl,enumfields,gitlab,psycopg2,requests,responses,rest_framework,rq,setuptools,sqlparse,teklia_toolbox,tenacity,tripoli,yaml diff --git a/Dockerfile b/Dockerfile index 6f0c06e349b7eef5a9eea9b818f315791b982caa..e7fec0e553b6796a4f77346e0043087d24d1f211 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,22 +5,12 @@ ADD . build RUN cd build && python3 setup.py sdist FROM registry.gitlab.com/arkindex/backend/base:django-3.1.7 -ARG COMMON_BRANCH=master -ARG COMMON_ID=9855787 ARG PONOS_BRANCH=master ARG PONOS_ID=10017043 ARG TRANSKRIBUS_BRANCH=master ARG TRANSKRIBUS_ID=11180199 ARG GITLAB_TOKEN="gaFM7LRa9zy9QMowcUhx" -# Install arkindex-common from private repo -RUN \ - mkdir /tmp/common && \ - wget --header "PRIVATE-TOKEN: $GITLAB_TOKEN" https://gitlab.com/api/v4/projects/$COMMON_ID/repository/archive.tar.gz?sha=$COMMON_BRANCH -O /tmp/common/archive.tar.gz && \ - tar --strip-components=1 -xvf /tmp/common/archive.tar.gz -C /tmp/common && \ - cd /tmp/common && pip install --disable-pip-version-check --no-cache-dir --quiet . && \ - rm -rf /tmp/common - # Install arkindex-ponos from private repo RUN \ mkdir /tmp/ponos && \ diff --git a/Dockerfile.binary b/Dockerfile.binary index f8cee8314c240744954a4788a90742d3a69a61d2..3d6ba8af3599fe55f18173b56277844529b34e98 100644 --- a/Dockerfile.binary +++ b/Dockerfile.binary @@ -4,8 +4,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y build-essential RUN pip install nuitka -ARG COMMON_BRANCH=master -ARG COMMON_ID=9855787 ARG PONOS_BRANCH=master ARG PONOS_ID=10017043 ARG TRANSKRIBUS_BRANCH=master @@ -21,12 +19,6 @@ ADD arkindex /usr/share/arkindex ADD base/requirements.txt /tmp/requirements-base-arkindex.txt ADD requirements.txt /tmp/requirements-arkindex.txt -# Retrieve common source code -RUN mkdir /tmp/common && \ - wget --header "PRIVATE-TOKEN: $GITLAB_TOKEN" https://gitlab.com/api/v4/projects/$COMMON_ID/repository/archive.tar.gz?sha=$COMMON_BRANCH -O /tmp/common.tar.gz && \ - tar --strip-components=1 -xvf /tmp/common.tar.gz -C /tmp/common && \ - mv /tmp/common/arkindex_common /usr/share - # Retrieve ponos source code RUN mkdir /tmp/ponos && \ wget --header "PRIVATE-TOKEN: $GITLAB_TOKEN" https://gitlab.com/api/v4/projects/$PONOS_ID/repository/archive.tar.gz?sha=$PONOS_BRANCH -O /tmp/ponos.tar.gz && \ @@ -43,7 +35,7 @@ RUN \ # Build full requirements, removing relative or remote references to arkindex projects # Special case for apistar, where we want to keep our own fork # Special case for approximate requirements from ponos, where we want to keep the versions specified from this repo -RUN cat /tmp/common/requirements.txt /tmp/ponos/requirements-server.txt /tmp/requirements-*arkindex.txt | sort | uniq | grep -v -E '^apistar|arkindex|^#|^Django~=|^boto3~=|^cryptography~=|^django-enumfields~=|^djangorestframework~=|^pyyaml~=|transkribus-client' > /requirements.txt +RUN cat /tmp/ponos/requirements-server.txt /tmp/requirements-*arkindex.txt | sort | uniq | grep -v -E '^apistar|arkindex|^#|^Django~=|^boto3~=|^cryptography~=|^django-enumfields~=|^djangorestframework~=|^pyyaml~=|transkribus-client' > /requirements.txt # List all management commands RUN find /usr/share/arkindex/*/management -name '*.py' -not -name '__init__.py' > /commands.txt @@ -56,7 +48,6 @@ RUN find /usr/share/arkindex /usr/share/ponos -type d -name tests | xargs rm -rf RUN python -m nuitka \ --nofollow-imports \ --include-package=arkindex \ - --include-package=arkindex_common \ --include-package=ponos \ --include-package=transkribus \ --show-progress \ diff --git a/Makefile b/Makefile index 1af263ad8d1d3af189f49a6594e6b26b739dbc84..fac7c4e5858b8c540f0a3646d54c30ecbf0efdb1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) PONOS_BRANCH=master -COMMON_BRANCH=master IMAGE_TAG=registry.gitlab.com/arkindex/backend .PHONY: all release @@ -15,10 +14,10 @@ clean: find . -name '*.pyc' -exec rm {} \; build: - CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) COMMON_BRANCH=$(COMMON_BRANCH) PONOS_BRANCH=$(PONOS_BRANCH) $(ROOT_DIR)/ci/build.sh Dockerfile + CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) PONOS_BRANCH=$(PONOS_BRANCH) $(ROOT_DIR)/ci/build.sh Dockerfile binary: - CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) COMMON_BRANCH=$(COMMON_BRANCH) PONOS_BRANCH=$(PONOS_BRANCH) $(ROOT_DIR)/ci/build.sh Dockerfile.binary -binary + CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) PONOS_BRANCH=$(PONOS_BRANCH) $(ROOT_DIR)/ci/build.sh Dockerfile.binary -binary worker: arkindex/manage.py rqworker -v 2 default high diff --git a/README.md b/README.md index 715d73f69dd606c9318e82c2acfbbbc3709cc16b..e306e3441a5f855144c6df404ff1dca43fa44d31 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,6 @@ SHELL_PLUS_POST_IMPORTS = [ 'PageDirection', 'PageComplement', )), - ('arkindex_common.enums', '*'), ('arkindex.dataimport.models', ( 'DataImportMode', )), diff --git a/arkindex/dataimport/api.py b/arkindex/dataimport/api.py index 2b5d5aaae154cd20c4b7c61a579f360c8db87416..cc074d2b6682f8d7051d58956bd1f3f8943d85b7 100644 --- a/arkindex/dataimport/api.py +++ b/arkindex/dataimport/api.py @@ -24,6 +24,7 @@ from rest_framework.views import APIView from arkindex.dataimport.models import ( DataFile, DataImport, + DataImportMode, Repository, RepositoryType, Revision, @@ -63,7 +64,6 @@ from arkindex.project.mixins import ( from arkindex.project.permissions import IsVerified, IsVerifiedOrReadOnly from arkindex.users.models import OAuthCredentials, Role, User from arkindex.users.utils import get_max_level -from arkindex_common.enums import DataImportMode from ponos.models import STATES_ORDERING, State logger = logging.getLogger(__name__) diff --git a/arkindex/dataimport/management/commands/import_s3.py b/arkindex/dataimport/management/commands/import_s3.py index 7f6731b5e786cdeafcb02e2770e3a169837d9a6c..9f951b820d6941fe9f57218597350c776393c71e 100644 --- a/arkindex/dataimport/management/commands/import_s3.py +++ b/arkindex/dataimport/management/commands/import_s3.py @@ -5,10 +5,9 @@ import yaml from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from arkindex.dataimport.models import DataImport +from arkindex.dataimport.models import DataImport, DataImportMode from arkindex.project.argparse import CorpusArgument from arkindex.users.models import User -from arkindex_common.enums import DataImportMode from ponos.models import Workflow logging.basicConfig( diff --git a/arkindex/dataimport/migrations/0001_initial.py b/arkindex/dataimport/migrations/0001_initial.py index 181f645b0ace8fe8b2ef61aff22344d581e6e533..8325ced57ce92c595d9ae3d1aa6e1def4655cd7e 100644 --- a/arkindex/dataimport/migrations/0001_initial.py +++ b/arkindex/dataimport/migrations/0001_initial.py @@ -10,7 +10,6 @@ from enumfields import Enum import arkindex.dataimport.models import arkindex.project.aws import arkindex.project.fields -import arkindex_common.enums class EventType(Enum): @@ -48,7 +47,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(auto_now_add=True)), ('updated', models.DateTimeField(auto_now=True)), - ('mode', enumfields.fields.EnumField(enum=arkindex_common.enums.DataImportMode, max_length=30)), + ('mode', enumfields.fields.EnumField(enum=arkindex.dataimport.models.DataImportMode, max_length=30)), ('payload', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)), ], options={ diff --git a/arkindex/dataimport/migrations/0011_refactor_payload_dataimport.py b/arkindex/dataimport/migrations/0011_refactor_payload_dataimport.py index f81b77aeeb07afe1fa366081bf5276b66a102893..7ff7be8bb70e2abc02d245e4f0a74a2e424a8ecb 100644 --- a/arkindex/dataimport/migrations/0011_refactor_payload_dataimport.py +++ b/arkindex/dataimport/migrations/0011_refactor_payload_dataimport.py @@ -3,7 +3,7 @@ import django.db.models.deletion from django.db import migrations, models -import arkindex_common +from arkindex.dataimport.models import DataImportMode def populate_new_fields(apps, schema_editor): @@ -17,7 +17,7 @@ def populate_new_fields(apps, schema_editor): ElementType = apps.get_model('documents', 'ElementType') Element = apps.get_model('documents', 'Element') for di in DataImport.objects.using(db_alias).filter( - mode=arkindex_common.enums.DataImportMode.Elements + mode=DataImportMode.Elements ): # There, we retrieve the payload elements that interest us to populate the new fields. We also remove the # attributes "ml_tools", "thumbnails" and "chunks" from the payload because they will no longer be useful. diff --git a/arkindex/dataimport/models.py b/arkindex/dataimport/models.py index c855916680f28f35b55c1c78373152f38573430b..4b669f536a14e2911248b0ada43f3da899552461 100644 --- a/arkindex/dataimport/models.py +++ b/arkindex/dataimport/models.py @@ -18,10 +18,19 @@ from arkindex.documents.models import ClassificationState, Element from arkindex.project.aws import S3FileMixin, S3FileStatus from arkindex.project.fields import ArrayField from arkindex.project.models import IndexableModel -from arkindex_common.enums import DataImportMode from ponos.models import Artifact, State, Workflow +class DataImportMode(Enum): + Images = 'images' + PDF = 'pdf' + Repository = 'repository' + Elements = 'elements' + IIIF = 'iiif' + Workers = 'workers' + Transkribus = 'transkribus' + + class DataImport(IndexableModel): """ A single import workflow diff --git a/arkindex/dataimport/serializers/imports.py b/arkindex/dataimport/serializers/imports.py index 74902c573486ac7ee178857ced28ec04e6a4c51f..64603c2509d66cd1e3614f2a1f41602907019310 100644 --- a/arkindex/dataimport/serializers/imports.py +++ b/arkindex/dataimport/serializers/imports.py @@ -4,13 +4,12 @@ from django.conf import settings from django.db.models import Q from rest_framework import serializers -from arkindex.dataimport.models import DataFile, DataImport, WorkerRun +from arkindex.dataimport.models import DataFile, DataImport, DataImportMode, WorkerRun from arkindex.dataimport.serializers.git import RevisionSerializer from arkindex.dataimport.serializers.workers import WorkerLightSerializer from arkindex.documents.models import Classification, ClassificationState, Corpus, Element, ElementType from arkindex.documents.serializers.elements import ElementSlimSerializer from arkindex.project.serializer_fields import BestClassField, EnumField -from arkindex_common.enums import DataImportMode from ponos.models import State from transkribus import TranskribusAPI diff --git a/arkindex/dataimport/tests/test_import_s3.py b/arkindex/dataimport/tests/test_import_s3.py index 4e618aee22d0a52bddf1194ca97b973b140ae276..caddec219426901cfafcada9fca92a371600358b 100644 --- a/arkindex/dataimport/tests/test_import_s3.py +++ b/arkindex/dataimport/tests/test_import_s3.py @@ -3,9 +3,8 @@ from django.core.management import call_command from django.core.management.base import CommandError from django.test import override_settings -from arkindex.dataimport.models import DataImport +from arkindex.dataimport.models import DataImport, DataImportMode from arkindex.project.tests import FixtureTestCase -from arkindex_common.enums import DataImportMode class TestImportS3(FixtureTestCase): diff --git a/arkindex/dataimport/tests/test_imports.py b/arkindex/dataimport/tests/test_imports.py index 045e9a5323069fc3f4a3ae92e0c39a755e0101bc..377af5a1f1fee7a115b24d1dd508c09e99fa8c52 100644 --- a/arkindex/dataimport/tests/test_imports.py +++ b/arkindex/dataimport/tests/test_imports.py @@ -7,11 +7,10 @@ from django.test import override_settings from django.urls import reverse from rest_framework import status -from arkindex.dataimport.models import DataImport, RepositoryType +from arkindex.dataimport.models import DataImport, DataImportMode, RepositoryType from arkindex.documents.models import Corpus, ElementType from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role, User -from arkindex_common.enums import DataImportMode from ponos.models import State, Task, Workflow RECIPE = ''' @@ -487,7 +486,7 @@ class TestImports(FixtureAPITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertDictEqual( response.json(), - {'__all__': ['Only processes of mode DataImportMode.Workers can be updated']} + {'__all__': ['Only processes of mode Workers can be updated']} ) def test_update_process_only_name(self): diff --git a/arkindex/dataimport/tests/test_repos.py b/arkindex/dataimport/tests/test_repos.py index 5daddc5f5d31ce2eed80489325d22c5c90b014bb..fc6b3751e2625de7dcd193eba778a2d4be8edc2c 100644 --- a/arkindex/dataimport/tests/test_repos.py +++ b/arkindex/dataimport/tests/test_repos.py @@ -5,10 +5,9 @@ from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.serializers import DateTimeField -from arkindex.dataimport.models import DataImport, Repository, RepositoryType +from arkindex.dataimport.models import DataImport, DataImportMode, Repository, RepositoryType from arkindex.project.tests import FixtureTestCase from arkindex.users.models import Role, User -from arkindex_common.enums import DataImportMode from ponos.models import State, Workflow RECIPE = ''' diff --git a/arkindex/dataimport/tests/test_signals.py b/arkindex/dataimport/tests/test_signals.py index 287026fe5ce30c364fcb38edf2704006eedf0d88..2a62986f68828be327b47aeb1ebd5c255997339b 100644 --- a/arkindex/dataimport/tests/test_signals.py +++ b/arkindex/dataimport/tests/test_signals.py @@ -1,9 +1,8 @@ from rest_framework.exceptions import ValidationError -from arkindex.dataimport.models import RepositoryType, Worker, WorkerRun, WorkerVersion +from arkindex.dataimport.models import DataImportMode, RepositoryType, Worker, WorkerRun, WorkerVersion from arkindex.dataimport.signals import _list_ancestors from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import DataImportMode class TestSignals(FixtureAPITestCase): diff --git a/arkindex/dataimport/tests/test_transkribus_import.py b/arkindex/dataimport/tests/test_transkribus_import.py index 100cd7d112e561779cddf50c39242703d351f8ac..a2cbdffa2ce9e9857ff7200e8214a2aa6cc7083e 100644 --- a/arkindex/dataimport/tests/test_transkribus_import.py +++ b/arkindex/dataimport/tests/test_transkribus_import.py @@ -6,11 +6,10 @@ from django.test import override_settings from django.urls import reverse from rest_framework import status -from arkindex.dataimport.models import DataImport +from arkindex.dataimport.models import DataImport, DataImportMode from arkindex.project.default_corpus import DEFAULT_TRANSKRIBUS_TYPES from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role, User -from arkindex_common.enums import DataImportMode from ponos.models import State diff --git a/arkindex/dataimport/tests/test_workerruns.py b/arkindex/dataimport/tests/test_workerruns.py index 147f4711640e5d70a17b8d580cdb97cc61c7a2e8..017deb88c3a927ced8e9d1b1f829413e15b63c06 100644 --- a/arkindex/dataimport/tests/test_workerruns.py +++ b/arkindex/dataimport/tests/test_workerruns.py @@ -4,10 +4,9 @@ from unittest.mock import patch from django.urls import reverse from rest_framework import status -from arkindex.dataimport.models import WorkerRun, WorkerVersion +from arkindex.dataimport.models import DataImportMode, WorkerRun, WorkerVersion from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role -from arkindex_common.enums import DataImportMode from ponos.models import Workflow RECIPE = ''' diff --git a/arkindex/dataimport/tests/test_workflows_api.py b/arkindex/dataimport/tests/test_workflows_api.py index de2a5f8f4efa7f9fa0581feb2f0a86c5f154b854..6fd28c4b3aac5ec9d1e54af834c4b840c9900189 100644 --- a/arkindex/dataimport/tests/test_workflows_api.py +++ b/arkindex/dataimport/tests/test_workflows_api.py @@ -3,10 +3,9 @@ from django.test import override_settings from rest_framework import status from rest_framework.reverse import reverse -from arkindex.dataimport.models import DataImport, RepositoryType, WorkerVersion +from arkindex.dataimport.models import DataImport, DataImportMode, RepositoryType, WorkerVersion from arkindex.documents.models import Corpus, Element from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import DataImportMode from ponos.models import State, Workflow RECIPE = ''' diff --git a/arkindex/documents/management/commands/build_fixtures.py b/arkindex/documents/management/commands/build_fixtures.py index d66517501919f796d68f261b60df6d8770df071e..d982fd3363c3cee92d556e734d449548f9a76f4c 100644 --- a/arkindex/documents/management/commands/build_fixtures.py +++ b/arkindex/documents/management/commands/build_fixtures.py @@ -7,10 +7,9 @@ from django.core.management.base import BaseCommand from django.utils import timezone as DjangoTimeZone from arkindex.dataimport.models import RepositoryType, WorkerVersion, WorkerVersionState, Workflow -from arkindex.documents.models import Corpus, Element, MetaData +from arkindex.documents.models import Corpus, Element, MetaData, MetaType from arkindex.images.models import Image, ImageServer, Zone from arkindex.users.models import Group, Right, Role, User -from arkindex_common.enums import MetaType from ponos.models import State diff --git a/arkindex/documents/migrations/0001_initial.py b/arkindex/documents/migrations/0001_initial.py index 3e3e3aaac52cd4010bb0e80dc771463a3f8543bd..4a5d9cfbe280c7341806583e6c06c963dc871040 100644 --- a/arkindex/documents/migrations/0001_initial.py +++ b/arkindex/documents/migrations/0001_initial.py @@ -11,7 +11,6 @@ from enumfields import Enum import arkindex.documents.dates import arkindex.documents.models import arkindex.project.fields -import arkindex_common.enums class Migration(migrations.Migration): @@ -34,7 +33,7 @@ class Migration(migrations.Migration): name='AllowedMetaData', fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('type', enumfields.fields.EnumField(enum=arkindex_common.enums.MetaType, max_length=50)), + ('type', enumfields.fields.EnumField(enum=arkindex.documents.models.MetaType, max_length=50)), ('name', models.CharField(max_length=250)), ], options={ @@ -109,7 +108,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('name', models.TextField()), - ('type', enumfields.fields.EnumField(db_index=True, enum=arkindex_common.enums.EntityType, max_length=50)), + ('type', enumfields.fields.EnumField(db_index=True, enum=arkindex.documents.models.EntityType, max_length=50)), ('metas', django.contrib.postgres.fields.hstore.HStoreField(blank=True, null=True)), ('validated', models.BooleanField(default=False)), ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='documents.Corpus')), @@ -214,7 +213,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('name', models.CharField(max_length=250)), - ('type', enumfields.fields.EnumField(db_index=True, enum=arkindex_common.enums.MetaType, max_length=50)), + ('type', enumfields.fields.EnumField(db_index=True, enum=arkindex.documents.models.MetaType, max_length=50)), ('value', models.TextField()), ('index', models.PositiveIntegerField(default=0)), ('element', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metadatas', to='documents.Element')), @@ -232,8 +231,8 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('parent_name', models.CharField(max_length=250)), ('child_name', models.CharField(max_length=250)), - ('parent_type', enumfields.fields.EnumField(enum=arkindex_common.enums.EntityType, max_length=50)), - ('child_type', enumfields.fields.EnumField(enum=arkindex_common.enums.EntityType, max_length=50)), + ('parent_type', enumfields.fields.EnumField(enum=arkindex.documents.models.EntityType, max_length=50)), + ('child_type', enumfields.fields.EnumField(enum=arkindex.documents.models.EntityType, max_length=50)), ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='roles', to='documents.Corpus')), ], ), diff --git a/arkindex/documents/models.py b/arkindex/documents/models.py index 8d5ebce41b88be4b73370e0ca32eae925ae5003e..bf9206602c68750874304ceb44869e103c87ef12 100644 --- a/arkindex/documents/models.py +++ b/arkindex/documents/models.py @@ -18,7 +18,6 @@ from arkindex.project.default_corpus import DEFAULT_CORPUS_TYPES, DEFAULT_TRANSK from arkindex.project.elastic import ESElement, ESEntity, ESTranscription from arkindex.project.fields import ArrayField from arkindex.project.models import IndexableModel -from arkindex_common.enums import EntityType, MetaType logger = logging.getLogger(__name__) @@ -260,6 +259,16 @@ class Element(IndexableModel): return '{}: {}'.format(self.type.display_name, self.name) +class EntityType(Enum): + Person = 'person' + Location = 'location' + Subject = 'subject' + Organization = 'organization' + Misc = 'misc' + Number = 'number' + Date = 'date' + + class Entity(InterpretedDateMixin, models.Model): """ Semantic object in arkindex @@ -494,6 +503,15 @@ class Classification(models.Model): ] +class MetaType(Enum): + Text = 'text' + HTML = 'html' + Date = 'date' + Location = 'location' + # Element's original structure reference (intended to be indexed) + Reference = 'reference' + + class AllowedMetaData(models.Model): """ Defines standard metadata's (type, name) couples for a corpus diff --git a/arkindex/documents/serializers/elements.py b/arkindex/documents/serializers/elements.py index 225380f927011c19b73235a3a7112baef60fd6e9..428e6dba26a081686445d87ded481c6485b86f3c 100644 --- a/arkindex/documents/serializers/elements.py +++ b/arkindex/documents/serializers/elements.py @@ -16,6 +16,7 @@ from arkindex.documents.models import ( ElementType, Entity, MetaData, + MetaType, Transcription, TranscriptionEntity, ) @@ -32,7 +33,6 @@ from arkindex.images.serializers import ZoneSerializer from arkindex.project.serializer_fields import LinearRingField from arkindex.users.models import Role from arkindex.users.utils import get_max_level -from arkindex_common.enums import MetaType class MetaDataSerializer(MetaDataLightSerializer): diff --git a/arkindex/documents/serializers/entities.py b/arkindex/documents/serializers/entities.py index 932cdf7550c6f6168fd4742d640d2d77b9b98b5d..9583c6f832953827a88c6c64a769321208fb462f 100644 --- a/arkindex/documents/serializers/entities.py +++ b/arkindex/documents/serializers/entities.py @@ -1,11 +1,10 @@ from rest_framework import serializers from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Corpus, Entity, EntityLink, EntityRole, TranscriptionEntity +from arkindex.documents.models import Corpus, Entity, EntityLink, EntityRole, EntityType, TranscriptionEntity from arkindex.documents.serializers.light import CorpusLightSerializer, InterpretedDateSerializer from arkindex.project.serializer_fields import EnumField from arkindex.project.triggers import reindex_start -from arkindex_common.enums import EntityType class BaseEntitySerializer(serializers.ModelSerializer): diff --git a/arkindex/documents/serializers/iiif/manifests.py b/arkindex/documents/serializers/iiif/manifests.py index e1bb717997cf5948ae6ea1ae13a294fec6f011be..7ab2a1aa4826e8c774b2794686ef310d84470ddf 100644 --- a/arkindex/documents/serializers/iiif/manifests.py +++ b/arkindex/documents/serializers/iiif/manifests.py @@ -3,10 +3,9 @@ from django.db.models import CharField, F, OuterRef, Prefetch, Subquery, Value from django.db.models.functions import Ceil, Concat from rest_framework import serializers -from arkindex.documents.models import Classification, Element, MetaData +from arkindex.documents.models import Classification, Element, MetaData, MetaType from arkindex.images.models import Image, Zone from arkindex.project.tools import bounding_box, build_absolute_url -from arkindex_common.enums import MetaType class ImageResourceManifestSerializer(serializers.BaseSerializer): diff --git a/arkindex/documents/serializers/light.py b/arkindex/documents/serializers/light.py index 77196d74b4253c0a8042391e81215380aa9efa4f..188e9f11c6eb70b2e72e9da235bd8e1a2c4f3570 100644 --- a/arkindex/documents/serializers/light.py +++ b/arkindex/documents/serializers/light.py @@ -4,11 +4,10 @@ from rest_framework.exceptions import APIException, ValidationError from arkindex.dataimport.serializers.git import RevisionSerializer from arkindex.documents.dates import DateType -from arkindex.documents.models import AllowedMetaData, Corpus, Element, ElementType, MetaData +from arkindex.documents.models import AllowedMetaData, Corpus, Element, ElementType, MetaData, MetaType from arkindex.images.serializers import ZoneLightSerializer from arkindex.project.serializer_fields import EnumField from arkindex.project.triggers import reindex_start -from arkindex_common.enums import MetaType class InterpretedDateSerializer(serializers.Serializer): diff --git a/arkindex/documents/serializers/search.py b/arkindex/documents/serializers/search.py index 1c331c744c7ab3d29b40ed5c857e4e781c006625..268739ae55db251cb114e7495e22c0225d45ee2f 100644 --- a/arkindex/documents/serializers/search.py +++ b/arkindex/documents/serializers/search.py @@ -4,12 +4,11 @@ from django.conf import settings from rest_framework import serializers from arkindex.documents.date_parser import parse_date -from arkindex.documents.models import Element, ElementType, Entity +from arkindex.documents.models import Element, ElementType, Entity, EntityType from arkindex.documents.serializers.light import CorpusLightSerializer, ElementLightSerializer from arkindex.documents.serializers.ml import TranscriptionSerializer from arkindex.images.serializers import ZoneSerializer from arkindex.project.serializer_fields import EnumField -from arkindex_common.enums import EntityType class SearchQuerySerializer(serializers.Serializer): diff --git a/arkindex/documents/tests/commands/test_reindex.py b/arkindex/documents/tests/commands/test_reindex.py index ff2d90d73e49745d10b8872ebda08c411cfd5daa..fd78ff99cadb470f996c0c20de0e1f059d6bd860 100644 --- a/arkindex/documents/tests/commands/test_reindex.py +++ b/arkindex/documents/tests/commands/test_reindex.py @@ -4,10 +4,9 @@ from django.core.management import CommandError, call_command from django.test import override_settings from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Element, Entity, EntityType, Transcription +from arkindex.documents.models import Element, Entity, EntityType, MetaType, Transcription from arkindex.project.elastic import ESElement, ESEntity, ESTranscription from arkindex.project.tests import FixtureTestCase -from arkindex_common.enums import MetaType class TestReindexCommand(FixtureTestCase): diff --git a/arkindex/documents/tests/tasks/test_corpus_delete.py b/arkindex/documents/tests/tasks/test_corpus_delete.py index b158d44dd91f7a4a98ce9a80cfaba51ff7ee15e0..946ef09424d3fa7129f24f023620f276d0a43182 100644 --- a/arkindex/documents/tests/tasks/test_corpus_delete.py +++ b/arkindex/documents/tests/tasks/test_corpus_delete.py @@ -1,10 +1,9 @@ from django.db.models.signals import pre_delete -from arkindex.dataimport.models import Repository, RepositoryType, WorkerVersion -from arkindex.documents.models import Corpus, Element, Transcription +from arkindex.dataimport.models import DataImportMode, Repository, RepositoryType, WorkerVersion +from arkindex.documents.models import Corpus, Element, EntityType, MetaType, Transcription from arkindex.documents.tasks import corpus_delete from arkindex.project.tests import FixtureTestCase -from arkindex_common.enums import DataImportMode, EntityType, MetaType class TestDeleteCorpus(FixtureTestCase): diff --git a/arkindex/documents/tests/tasks/test_reindex.py b/arkindex/documents/tests/tasks/test_reindex.py index 552f169965cb89c13ac40a1281aaa5a3d5f6278e..4ff9d96fd91fe63b17671c3c2c81ec4a1a74184c 100644 --- a/arkindex/documents/tests/tasks/test_reindex.py +++ b/arkindex/documents/tests/tasks/test_reindex.py @@ -4,10 +4,9 @@ from django.contrib.gis.geos import LinearRing from django.db.models import Q from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Corpus, Element, Entity, Transcription +from arkindex.documents.models import Corpus, Element, Entity, EntityType, MetaType, Transcription from arkindex.documents.tasks import reindex_start from arkindex.project.tests import FixtureTestCase -from arkindex_common.enums import EntityType, MetaType @patch('arkindex.documents.tasks.Indexer') diff --git a/arkindex/documents/tests/test_entities.py b/arkindex/documents/tests/test_entities.py index 1f72233cc5c611976a9f9fcc691cbd818cef3593..a7b3558375b3861fdc659f582897d6dc1bd707e3 100644 --- a/arkindex/documents/tests/test_entities.py +++ b/arkindex/documents/tests/test_entities.py @@ -1,9 +1,8 @@ from django.core.exceptions import ValidationError from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Corpus, Entity, EntityLink, EntityRole, MetaData +from arkindex.documents.models import Corpus, Entity, EntityLink, EntityRole, EntityType, MetaData, MetaType from arkindex.project.tests import FixtureTestCase -from arkindex_common.enums import EntityType, MetaType class TestSaveEntities(FixtureTestCase): diff --git a/arkindex/documents/tests/test_entities_api.py b/arkindex/documents/tests/test_entities_api.py index 66f3adb0ec545e98a876386705bc5aa2faff1c7a..c03373882cf282a73edd376638fff0c90f47397f 100644 --- a/arkindex/documents/tests/test_entities_api.py +++ b/arkindex/documents/tests/test_entities_api.py @@ -8,9 +8,17 @@ from elasticsearch.exceptions import NotFoundError from rest_framework import status from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Corpus, Element, Entity, EntityLink, EntityRole, EntityType, TranscriptionEntity +from arkindex.documents.models import ( + Corpus, + Element, + Entity, + EntityLink, + EntityRole, + EntityType, + MetaType, + TranscriptionEntity, +) from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import MetaType class TestEntitiesAPI(FixtureAPITestCase): diff --git a/arkindex/documents/tests/test_indexer.py b/arkindex/documents/tests/test_indexer.py index 3981f62a9af86f10789f37e9df598e9313e7a5e1..494601ae6bc43744296fd31d04a0c14a111d1876 100644 --- a/arkindex/documents/tests/test_indexer.py +++ b/arkindex/documents/tests/test_indexer.py @@ -5,8 +5,8 @@ from elasticsearch.exceptions import NotFoundError from arkindex.dataimport.models import WorkerVersion from arkindex.documents.indexer import Indexer +from arkindex.documents.models import EntityType from arkindex.project.tests import FixtureTestCase -from arkindex_common.enums import EntityType class TestIndexer(FixtureTestCase): diff --git a/arkindex/documents/tests/test_manifest.py b/arkindex/documents/tests/test_manifest.py index 4eaafa569977824d86e33eac4e62473d26dc0dcf..701559d37bcbf5b9d5ab1a102a2d90dec034ce6d 100644 --- a/arkindex/documents/tests/test_manifest.py +++ b/arkindex/documents/tests/test_manifest.py @@ -5,9 +5,8 @@ from rest_framework import status from tripoli import IIIFValidator from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Element +from arkindex.documents.models import Element, MetaType from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import MetaType class TestFolderManifestSerializer(FixtureAPITestCase): diff --git a/arkindex/documents/tests/test_metadata.py b/arkindex/documents/tests/test_metadata.py index 9888f16c407a98ec0d20df68e016680bf92b2f83..9a116280efe66e24f92c7aa94b859f0aa0c1fa0b 100644 --- a/arkindex/documents/tests/test_metadata.py +++ b/arkindex/documents/tests/test_metadata.py @@ -5,10 +5,9 @@ from django.urls import reverse from rest_framework import status from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import AllowedMetaData, Corpus, MetaData +from arkindex.documents.models import AllowedMetaData, Corpus, EntityType, MetaData, MetaType from arkindex.project.tests import FixtureAPITestCase from arkindex.users.models import Role, User -from arkindex_common.enums import EntityType, MetaType class TestMetaData(FixtureAPITestCase): diff --git a/arkindex/documents/tests/test_retrieve_elements.py b/arkindex/documents/tests/test_retrieve_elements.py index fac30efe15ad558c3ef49164005938c3c934182f..cd0480c29388cf89befd5c2ad34a26589c535507 100644 --- a/arkindex/documents/tests/test_retrieve_elements.py +++ b/arkindex/documents/tests/test_retrieve_elements.py @@ -2,9 +2,8 @@ from django.urls import reverse from rest_framework import status from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Corpus, Entity, MLClass +from arkindex.documents.models import Corpus, Entity, EntityType, MetaType, MLClass from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import EntityType, MetaType class TestRetrieveElements(FixtureAPITestCase): diff --git a/arkindex/documents/tests/test_search.py b/arkindex/documents/tests/test_search.py index 3068434d0c3860c65ec7a0c86b99ae520449904f..73d0158ed3c264ff524707fd1cc9582bbadab62e 100644 --- a/arkindex/documents/tests/test_search.py +++ b/arkindex/documents/tests/test_search.py @@ -7,10 +7,9 @@ from elasticsearch_dsl.connections import connections from rest_framework import status from arkindex.dataimport.models import WorkerVersion -from arkindex.documents.models import Corpus, Element, Transcription +from arkindex.documents.models import Corpus, Element, EntityType, MetaType, Transcription from arkindex.project.elastic import ESTranscription from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import EntityType, MetaType class TestSearchAPI(FixtureAPITestCase): diff --git a/arkindex/project/elastic.py b/arkindex/project/elastic.py index ab16b980595b378bfba0a1b4c31e54728b17d737..942d2381625442de4a6a2a26d9f5f570850a3a27 100644 --- a/arkindex/project/elastic.py +++ b/arkindex/project/elastic.py @@ -5,8 +5,6 @@ from django.conf import settings from elasticsearch_dsl import Date, DateRange, Document, Float, InnerDoc, Keyword, Mapping, Nested, Search, Text from elasticsearch_dsl.connections import connections -from arkindex_common.enums import MetaType - connections.create_connection(hosts=settings.ELASTIC_SEARCH_HOSTS) @@ -105,7 +103,7 @@ class ESElement(Document): @classmethod def from_model(cls, instance): - from arkindex.documents.models import Element + from arkindex.documents.models import Element, MetaType interpreted_dates = chain(*[md.get_dates() for md in instance.metadatas.all()]) date_range = { diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index 87176e7d45d2c7a095a4cb938d0651439069a256..dafa596e748517599da6c8b157c75fe211073b81 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -237,10 +237,9 @@ SPECTACULAR_SETTINGS = { # Spectacular can warn about duplicates. # To fix this, the entries below map custom names to each affected enum. 'ENUM_NAME_OVERRIDES': { - 'MLToolType': 'arkindex_common.ml_tool.MLToolType', - 'MetaType': 'arkindex_common.enums.MetaType', - 'ProcessMode': 'arkindex_common.enums.DataImportMode', - 'EntityType': 'arkindex_common.enums.EntityType', + 'MetaType': 'arkindex.documents.models.MetaType', + 'ProcessMode': 'arkindex.dataimport.models.DataImportMode', + 'EntityType': 'arkindex.documents.models.EntityType', 'S3FileStatus': 'arkindex.project.aws.S3FileStatus', 'ClassificationState': 'arkindex.documents.models.ClassificationState', 'PonosState': 'ponos.models.State', diff --git a/arkindex/project/tests/test_elastic.py b/arkindex/project/tests/test_elastic.py index 3a9da527cdd6aa2404319dbadad7ba6d24d4d2f1..e675e8be53ab00e168f00ab8c508e0a6a23333e7 100644 --- a/arkindex/project/tests/test_elastic.py +++ b/arkindex/project/tests/test_elastic.py @@ -2,9 +2,9 @@ from unittest.mock import patch from arkindex.dataimport.models import WorkerVersion from arkindex.documents.dates import DateType, InterpretedDate +from arkindex.documents.models import MetaType from arkindex.project.elastic import ESElement from arkindex.project.tests import FixtureAPITestCase -from arkindex_common.enums import MetaType class TestESDocuments(FixtureAPITestCase): diff --git a/ci/build.sh b/ci/build.sh index 246c701de963225fb2a0b7f22999bd2597b2e5fe..517131802f1bb7a3abbe8b4e17b7c32f818061c8 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -38,12 +38,11 @@ if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ] fi PONOS_BRANCH=${PONOS_BRANCH:-master} -COMMON_BRANCH=${COMMON_BRANCH:-master} IMAGE_TAG="$CI_REGISTRY_IMAGE:$VERSION" cd $CI_PROJECT_DIR docker pull "$CI_REGISTRY_IMAGE/base:latest" -docker build . -f $DOCKERFILE -t "$IMAGE_TAG" --build-arg "PONOS_BRANCH=$PONOS_BRANCH" --build-arg "COMMON_BRANCH=$COMMON_BRANCH" +docker build . -f $DOCKERFILE -t "$IMAGE_TAG" --build-arg "PONOS_BRANCH=$PONOS_BRANCH" if [ -n "$CI_REGISTRY" ]; then docker push "$IMAGE_TAG" fi diff --git a/requirements.txt b/requirements.txt index 7c4da881d91f401d9edc491d75778bdeaa418732..b4f79f6d705fded0d9cacd4e8710e4ef62fc8b33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,6 @@ apistar==0.7.2 git+https://gitlab.com/teklia/apistar.git#egg=apistar -arkindex-common==0.2.0 -git+https://gitlab.com/arkindex/common.git#egg=arkindex-common django-admin-hstore-widget==1.1.0 django-cors-headers==3.7.0 django-enumfields==2.0.0