From f11d06230a18c00335f9146b1b926e32eaedafd3 Mon Sep 17 00:00:00 2001
From: manonBlanco <blanco@teklia.com>
Date: Fri, 13 Dec 2019 14:00:14 +0100
Subject: [PATCH] delete metatype entity
---
arkindex/documents/consumers.py | 3 --
.../0030_remove_entity_type_metadata.py | 28 +++++++++++++++++++
arkindex/documents/serializers/elements.py | 2 +-
arkindex/documents/tei.py | 2 +-
.../consumers/test_ml_results_consumer.py | 10 +++----
.../tests/consumers/test_reindex_consumer.py | 2 +-
arkindex/documents/tests/test_entities.py | 2 +-
arkindex/documents/tests/test_entities_api.py | 4 +--
arkindex/documents/tests/test_tei.py | 12 ++++----
9 files changed, 45 insertions(+), 20 deletions(-)
create mode 100644 arkindex/documents/migrations/0030_remove_entity_type_metadata.py
diff --git a/arkindex/documents/consumers.py b/arkindex/documents/consumers.py
index 49723dc9b1..72ac063a38 100644
--- a/arkindex/documents/consumers.py
+++ b/arkindex/documents/consumers.py
@@ -3,7 +3,6 @@ from channels.consumer import SyncConsumer
from django.db.models import Q
from django.db.models.deletion import Collector
from django.db.models.signals import pre_delete
-from arkindex_common.enums import MetaType
from arkindex.project.tools import disconnect_signal
from arkindex.project.triggers import reindex_start
from arkindex.dataimport.models import Event, Revision
@@ -130,8 +129,6 @@ class MLResultsConsumer(SyncConsumer):
# Metadata are not considered ML results so we need to keep them: update them to unlink entities.
logger.info('Updating element metadata')
MetaData.objects.filter(element__in=elements).update(entity_id=None)
- # TODO: Remove MetaType.Entity entirely
- MetaData.objects.filter(element__in=elements, type=MetaType.Entity).update(type=MetaType.Text)
# We removed transcriptions earlier, which implies removing the links with entities.
# All is left is to remove 'lonely' entities.
diff --git a/arkindex/documents/migrations/0030_remove_entity_type_metadata.py b/arkindex/documents/migrations/0030_remove_entity_type_metadata.py
new file mode 100644
index 0000000000..1092bfbceb
--- /dev/null
+++ b/arkindex/documents/migrations/0030_remove_entity_type_metadata.py
@@ -0,0 +1,28 @@
+# Generated by Django 2.2 on 2019-12-13 10:28
+
+from django.db import migrations
+from arkindex_common.enums import MetaType
+
+
+def updateMetadata(apps, schema_editor):
+ values = set(item.value for item in MetaType)
+ MetaData = apps.get_model('documents', 'MetaData')
+ for metadata in MetaData.objects \
+ .filter(type=MetaType.Entity) \
+ .prefetch_related('entity'):
+ metadata_type = MetaType.Text
+ if metadata.entity and metadata.entity.type.value in values:
+ metadata_type = MetaType(metadata.entity.type.value)
+ metadata.type = metadata_type
+ metadata.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('documents', '0029_allowedmetadata'),
+ ]
+
+ operations = [
+ migrations.RunPython(updateMetadata, migrations.RunPython.noop),
+ ]
diff --git a/arkindex/documents/serializers/elements.py b/arkindex/documents/serializers/elements.py
index 2535ba9983..8a4493fdfe 100644
--- a/arkindex/documents/serializers/elements.py
+++ b/arkindex/documents/serializers/elements.py
@@ -229,7 +229,7 @@ class ElementSerializer(ElementSlimSerializer):
def get_metadata(self, obj):
return MetaDataLightSerializer(
- list(obj.metadatas.all().filter(entity__isnull=True).exclude(type=MetaType.Entity)),
+ list(obj.metadatas.all().filter(entity__isnull=True)),
many=True
).data
diff --git a/arkindex/documents/tei.py b/arkindex/documents/tei.py
index 0da96c338e..65f0ba82da 100644
--- a/arkindex/documents/tei.py
+++ b/arkindex/documents/tei.py
@@ -94,7 +94,7 @@ class TeiParser(BaseTeiParser):
for index, xmlmeta in enumerate(metadatas):
# Handle entities
entity = None
- if xmlmeta.type == MetaType.Entity and xmlmeta.entity_type is not None:
+ if xmlmeta.entity_type is not None:
source, _ = DataSource.objects.get_or_create(
type=MLToolType.NER,
slug='TEI',
diff --git a/arkindex/documents/tests/consumers/test_ml_results_consumer.py b/arkindex/documents/tests/consumers/test_ml_results_consumer.py
index 6e38dedce4..afa98755d1 100644
--- a/arkindex/documents/tests/consumers/test_ml_results_consumer.py
+++ b/arkindex/documents/tests/consumers/test_ml_results_consumer.py
@@ -36,7 +36,7 @@ class TestMLResultsConsumer(FixtureTestCase):
source=source,
)
cls.page1.metadatas.create(
- type=MetaType.Entity,
+ type=MetaType.Text,
name='something',
value='Some entity 1',
entity=entity1,
@@ -64,7 +64,7 @@ class TestMLResultsConsumer(FixtureTestCase):
source=source,
)
cls.page2.metadatas.create(
- type=MetaType.Entity,
+ type=MetaType.Text,
name='something',
value='Some entity 2',
entity=entity2,
@@ -104,7 +104,7 @@ class TestMLResultsConsumer(FixtureTestCase):
self.assertEqual(self.page1.metadatas.count(), 2)
self.assertEqual(self.page2.metadatas.count(), 2)
- with self.assertNumQueries(17):
+ with self.assertNumQueries(16):
MLResultsConsumer({}).ml_results_delete({'corpus_id': str(self.corpus.id)})
for queryset in querysets:
@@ -144,7 +144,7 @@ class TestMLResultsConsumer(FixtureTestCase):
for queryset in folder2_querysets:
self.assertTrue(queryset.exists())
- with self.assertNumQueries(18):
+ with self.assertNumQueries(17):
MLResultsConsumer({}).ml_results_delete({'element_id': str(self.folder1.id)})
for queryset in folder1_querysets:
@@ -158,7 +158,7 @@ class TestMLResultsConsumer(FixtureTestCase):
self.assertEqual(metadata.type, MetaType.Text)
self.assertIsNone(metadata.entity)
metadata = self.page2.metadatas.get(value='Some entity 2')
- self.assertEqual(metadata.type, MetaType.Entity)
+ self.assertEqual(metadata.type, MetaType.Text)
self.assertIsNotNone(metadata.entity)
get_layer_mock().send.assert_called_once_with('reindex', {
diff --git a/arkindex/documents/tests/consumers/test_reindex_consumer.py b/arkindex/documents/tests/consumers/test_reindex_consumer.py
index 44c41b98c9..6274f332c2 100644
--- a/arkindex/documents/tests/consumers/test_reindex_consumer.py
+++ b/arkindex/documents/tests/consumers/test_reindex_consumer.py
@@ -16,7 +16,7 @@ class TestReindexConsumer(FixtureTestCase):
cls.folder = cls.corpus.elements.get(name='Volume 1')
cls.folder.metadatas.create(
- type=MetaType.Entity,
+ type=MetaType.Text,
name='something',
value='Some entity',
entity=cls.corpus.entities.create(
diff --git a/arkindex/documents/tests/test_entities.py b/arkindex/documents/tests/test_entities.py
index 9bf10b6497..bd8d6ab612 100644
--- a/arkindex/documents/tests/test_entities.py
+++ b/arkindex/documents/tests/test_entities.py
@@ -70,7 +70,7 @@ class TestSaveEntities(FixtureTestCase):
with self.assertRaises(ValidationError):
MetaData.objects.create(
name='test 1',
- type=MetaType.Entity,
+ type=MetaType.Text,
value='Blah',
element=element,
entity=self.parent,
diff --git a/arkindex/documents/tests/test_entities_api.py b/arkindex/documents/tests/test_entities_api.py
index 2ee4d7ba68..13415c146c 100644
--- a/arkindex/documents/tests/test_entities_api.py
+++ b/arkindex/documents/tests/test_entities_api.py
@@ -50,7 +50,7 @@ class TestEntitiesAPI(FixtureAPITestCase):
)
self.metadata = self.page.metadatas.create(
name='test 1',
- type=MetaType.Entity,
+ type=MetaType.Text,
value='Blah',
element=self.page,
entity=self.entity,
@@ -485,7 +485,7 @@ class TestEntitiesAPI(FixtureAPITestCase):
"""
List an element's entities sorted depending on their origin (metadata or transcriptions)
"""
- md = self.page.metadatas.create(name='some_metadata', type=MetaType.Entity, value='something')
+ md = self.page.metadatas.create(name='some_metadata', type=MetaType.Location, value='something')
md.entity = self.entity_bis
md.save()
with self.assertNumQueries(10):
diff --git a/arkindex/documents/tests/test_tei.py b/arkindex/documents/tests/test_tei.py
index bb2a02bf92..da6c3c7fca 100644
--- a/arkindex/documents/tests/test_tei.py
+++ b/arkindex/documents/tests/test_tei.py
@@ -79,7 +79,7 @@ class TestTeiElement(FixtureTestCase):
{
'name': 'location',
'index': 0,
- 'type': MetaType.Entity,
+ 'type': MetaType.Location,
'value': 'Mars',
'entity__name': 'Mars',
'entity__type': EntityType.Location,
@@ -87,7 +87,7 @@ class TestTeiElement(FixtureTestCase):
{
'name': 'person',
'index': 1,
- 'type': MetaType.Entity,
+ 'type': MetaType.Text,
'value': 'The Chosen One',
'entity__name': 'The Chosen One',
'entity__type': EntityType.Person,
@@ -120,7 +120,7 @@ class TestTeiElement(FixtureTestCase):
{
'name': 'person',
'index': 0,
- 'type': MetaType.Entity,
+ 'type': MetaType.Text,
'value': 'Someone',
'entity__name': 'Someone',
'entity__type': EntityType.Person,
@@ -128,7 +128,7 @@ class TestTeiElement(FixtureTestCase):
{
'name': 'place',
'index': 0,
- 'type': MetaType.Entity,
+ 'type': MetaType.Location,
'value': 'Somewhere',
'entity__name': 'Somewhere',
'entity__type': EntityType.Location,
@@ -147,7 +147,7 @@ class TestTeiElement(FixtureTestCase):
{
'name': 'person',
'index': 0,
- 'type': MetaType.Entity,
+ 'type': MetaType.Text,
'value': 'Someone else',
'entity__name': 'Someone else',
'entity__type': EntityType.Person,
@@ -155,7 +155,7 @@ class TestTeiElement(FixtureTestCase):
{
'name': 'subject',
'index': 0,
- 'type': MetaType.Entity,
+ 'type': MetaType.Text,
'value': 'Something',
'entity__name': 'Something',
'entity__type': EntityType.Subject,
--
GitLab