Merge branch 'entity' into 'master'

Entity model

See merge request !302

arkindex/documents/admin.py

 from django.contrib import admin
+from django import forms
 from django.urls import path, reverse
 from django.utils.html import format_html
+from django_admin_hstore_widget.forms import HStoreFormField
 from arkindex.documents.models import \
-    Corpus, Page, Element, ElementType, Act, Transcription, MetaData, InterpretedDate, Classification, DataSource
+    Corpus, Page, Element, ElementType, Act, Transcription, MetaData, InterpretedDate, Classification, DataSource, \
+    Entity, EntityRole, EntityLink
 from arkindex.documents.views import DumpActs
 from arkindex.dataimport.models import Event
 from enumfields.admin import EnumFieldListFilter
@@ -43,12 +46,13 @@ class DateInline(admin.TabularInline):
 class MetaDataAdmin(admin.ModelAdmin):
     list_display = ('id', 'type', 'revision')
     readonly_fields = ('id', 'revision')
-    raw_id_fields = ('element', )
+    raw_id_fields = ('element', 'entity', )
     inlines = (DateInline, )
 
 
 class MetaDataInline(admin.TabularInline):
     model = MetaData
+    raw_id_fields = ('entity', )
 
 
 class ElementAdmin(admin.ModelAdmin):
@@ -95,6 +99,31 @@ class TranscriptionAdmin(admin.ModelAdmin):
     raw_id_fields = ('element', 'zone', )
 
 
+class EntityMetaForm(forms.ModelForm):
+    metas = HStoreFormField()
+
+
+class EntityLinkInLine(admin.TabularInline):
+    model = EntityLink
+    fk_name = 'parent'
+    raw_id_fields = ('child', )
+
+
+class EntityAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'type')
+    list_filter = [('type', EnumFieldListFilter), 'corpus']
+    readonly_fields = ('id', )
+    search_fields = ('name', )
+    inlines = (EntityLinkInLine, )
+    form = EntityMetaForm
+
+
+class EntityRoleAdmin(admin.ModelAdmin):
+    list_display = ('id', 'parent_name', 'child_name')
+    list_filter = ['corpus']
+    readonly_fields = ('id', )
+
+
 admin.site.register(Corpus, CorpusAdmin)
 admin.site.register(DataSource, DataSourceAdmin)
 admin.site.register(Page, PageAdmin)
@@ -102,3 +131,5 @@ admin.site.register(Element, ElementAdmin)
 admin.site.register(Act, ActAdmin)
 admin.site.register(Transcription, TranscriptionAdmin)
 admin.site.register(MetaData, MetaDataAdmin)
+admin.site.register(Entity, EntityAdmin)
+admin.site.register(EntityRole, EntityRoleAdmin)

arkindex/documents/migrations/0003_entities.py

+# Generated by Django 2.1 on 2019-05-02 13:40
+
+import arkindex.documents.models
+import django.contrib.postgres.fields.hstore
+from django.contrib.postgres.operations import HStoreExtension
+from django.db import migrations, models
+import django.db.models.deletion
+import enumfields.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0002_interpreteddate'),
+    ]
+
+    operations = [
+        HStoreExtension(),
+        migrations.CreateModel(
+            name='Entity',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('name', models.TextField()),
+                ('type', enumfields.fields.EnumField(
+                    db_index=True,
+                    enum=arkindex.documents.models.EntityType,
+                    max_length=50)),
+                ('metas', django.contrib.postgres.fields.hstore.HStoreField(blank=True, null=True)),
+                ('corpus', models.ForeignKey(
+                    on_delete=django.db.models.deletion.CASCADE,
+                    related_name='entities',
+                    to='documents.Corpus')),
+            ],
+            options={
+                'verbose_name_plural': 'Entities',
+            },
+        ),
+        migrations.CreateModel(
+            name='EntityLink',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('child', models.ForeignKey(
+                    on_delete=django.db.models.deletion.CASCADE,
+                    related_name='children',
+                    to='documents.Entity')),
+                ('parent', models.ForeignKey(
+                    on_delete=django.db.models.deletion.CASCADE,
+                    related_name='parents',
+                    to='documents.Entity')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='EntityRole',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('parent_name', models.CharField(max_length=250)),
+                ('child_name', models.CharField(max_length=250)),
+                ('parent_type', enumfields.fields.EnumField(enum=arkindex.documents.models.EntityType, max_length=50)),
+                ('child_type', enumfields.fields.EnumField(enum=arkindex.documents.models.EntityType, max_length=50)),
+                ('corpus', models.ForeignKey(
+                    on_delete=django.db.models.deletion.CASCADE,
+                    related_name='roles',
+                    to='documents.Corpus')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='entitylink',
+            name='role',
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name='links',
+                to='documents.EntityRole'),
+        ),
+        migrations.AddField(
+            model_name='metadata',
+            name='entity',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name='metadatas',
+                to='documents.Entity'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='entityrole',
+            unique_together={('parent_name', 'child_name', 'parent_type', 'child_type', 'corpus')},
+        ),
+        migrations.AlterUniqueTogether(
+            name='entity',
+            unique_together={('corpus', 'type', 'name')},
+        ),
+    ]

arkindex/documents/migrations/0004_entities_data_migration.py

+# Generated by Django 2.1 on 2019-05-02 13:18
+
+from django.db import migrations
+from django.db.models import Q, OuterRef, Subquery
+from arkindex_common.enums import MetaType
+from arkindex.documents.models import EntityType
+
+
+def createEntities(apps, schema_editor):
+    values = set(item.value for item in EntityType)
+    MetaData = apps.get_model('documents', 'MetaData')
+    Entity = apps.get_model('documents', 'Entity')
+    for metadata in MetaData.objects \
+            .filter(Q(type=MetaType.Text) & (Q(name__in=values) | Q(name='place'))) \
+            .prefetch_related('element'):
+        if metadata.name in values:
+            type = EntityType(metadata.name)
+        else:
+            type = EntityType.Location
+        entity, created = Entity.objects.get_or_create(
+            name=metadata.value,
+            type=type,
+            corpus_id=metadata.element.corpus_id)
+        if created:
+            entity.save()
+            print(entity)
+        metadata.value = ''
+        metadata.type = MetaType.Entity
+        metadata.entity = entity
+        metadata.save()
+
+
+def deleteEntities(apps, schema_editor):
+    MetaData = apps.get_model('documents', 'MetaData')
+    # Use Subquery() because Django doesn't support F() with joins
+    MetaData.objects \
+        .filter(entity__isnull=False) \
+        .update(
+            type=MetaType.Text,
+            value=Subquery(MetaData.objects.filter(pk=OuterRef('pk')).values('entity__name')[:1]))
+    Entity = apps.get_model('documents', 'Entity')
+    Entity.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0003_entities'),
+    ]
+
+    operations = [
+        migrations.RunPython(createEntities, deleteEntities),
+    ]

arkindex/documents/models.py

 from django.db import models, transaction
 from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.fields import HStoreField
 from django.utils.functional import cached_property
 from django.core.validators import MinValueValidator, MaxValueValidator
 from django.core.exceptions import ValidationError
@@ -581,6 +582,91 @@ class Classification(models.Model):
         )
 
 
+class EntityType(Enum):
+    Person = 'person'
+    Location = 'location'
+    Subject = 'subject'
+    Organization = 'organization'
+    Misc = 'misc'
+
+
+class Entity(models.Model):
+    """
+    Semantic object in arkindex
+    """
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    name = models.TextField()
+    type = EnumField(EntityType, max_length=50, db_index=True)
+    corpus = models.ForeignKey(Corpus, related_name='entities', on_delete=models.CASCADE)
+    metas = HStoreField(null=True, blank=True)
+
+    class Meta:
+        unique_together = (
+            ('corpus', 'type', 'name'),
+        )
+        verbose_name_plural = 'Entities'
+
+    def __str__(self):
+        return self.name
+
+
+class EntityRole(models.Model):
+    """
+    Role's type between a parent and a child
+    """
+    parent_name = models.CharField(max_length=250)
+    child_name = models.CharField(max_length=250)
+    parent_type = EnumField(EntityType, max_length=50)
+    child_type = EnumField(EntityType, max_length=50)
+    corpus = models.ForeignKey(Corpus, related_name='roles', on_delete=models.CASCADE)
+
+    class Meta:
+        unique_together = (
+            ('parent_name', 'child_name', 'parent_type', 'child_type', 'corpus'),
+        )
+
+    def __str__(self):
+        return '{} -> {}'.format(self.parent_name, self.child_name)
+
+
+class EntityLink(models.Model):
+    """
+    Link between two entities with a role
+    """
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    parent = models.ForeignKey(Entity, related_name='parents', on_delete=models.CASCADE)
+    child = models.ForeignKey(Entity, related_name='children', on_delete=models.CASCADE)
+    role = models.ForeignKey(EntityRole, related_name='links', on_delete=models.CASCADE)
+
+    def clean(self):
+        if self.role is None:
+            return
+        if self.parent is None:
+            return
+        if self.parent.type != self.role.parent_type:
+            raise ValidationError("Parent's type {} is different from the expected type {}".format(
+                self.parent.type,
+                self.role.parent_type))
+        if self.parent.corpus != self.role.corpus:
+            raise ValidationError("Parent's corpus {} is different from the expected corpus {}".format(
+                self.parent.corpus,
+                self.role.corpus))
+        if self.child is None:
+            return
+        if self.child.type != self.role.child_type:
+            raise ValidationError("Child's type {} is different from the expected type {}".format(
+                self.child.type,
+                self.role.child_type))
+        if self.child.corpus != self.role.corpus:
+            raise ValidationError("Child's corpus {} is different from the expected corpus {}".format(
+                self.child.corpus,
+                self.role.corpus))
+
+    def save(self, *args, **kwargs):
+        self.full_clean()
+        super().save(*args, **kwargs)
+
+
 class MetaData(models.Model):
     '''
     Metadatas for elements
@@ -592,6 +678,7 @@ class MetaData(models.Model):
     value = models.TextField()
     revision = models.ForeignKey('dataimport.Revision', on_delete=models.SET_NULL, blank=True, null=True)
     index = models.PositiveIntegerField(default=0)
+    entity = models.ForeignKey(Entity, null=True, blank=True, related_name='metadatas', on_delete=models.SET_NULL)
 
     class Meta:
         ordering = ('element', 'name', 'index')
@@ -624,6 +711,18 @@ class MetaData(models.Model):
     def __ge__(self, other):
         return self.__eq__(other) or self.__gt__(other)
 
+    def clean(self):
+        if self.entity is None or self.element is None:
+            return
+        if self.entity.corpus != self.element.corpus:
+            raise ValidationError("Entity's corpus {} is different from the expected corpus {}".format(
+                self.entity.corpus,
+                self.element.corpus))
+
+    def save(self, *args, **kwargs):
+        self.full_clean()
+        super().save(*args, **kwargs)
+
 
 class DateType(Enum):
     Exact = 'exact'

arkindex/documents/tests/test_entities.py

+from django.core.exceptions import ValidationError
+from arkindex.documents.models import Corpus, Element, ElementType, \
+    Entity, EntityType, EntityRole, EntityLink, MetaData, MetaType
+from arkindex.project.tests import FixtureTestCase
+
+
+class TestSaveEntities(FixtureTestCase):
+
+    @classmethod
+    def setUpTestData(cls):
+        super().setUpTestData()
+        cls.corpus1 = Corpus.objects.create(name='corpus 1')
+        cls.corpus2 = Corpus.objects.create(name='corpus 2')
+        cls.parent = Entity.objects.create(name='parent', type=EntityType.Organization, corpus=cls.corpus1)
+        cls.child = Entity.objects.create(type=EntityType.Person, corpus=cls.corpus1, name="child")
+        cls.role = EntityRole.objects.create(
+            parent_name='organization',
+            child_name='person',
+            parent_type=EntityType.Organization,
+            child_type=EntityType.Person,
+            corpus=cls.corpus1,
+        )
+        cls.link = EntityLink(parent=cls.parent, child=cls.child, role=cls.role)
+
+    def test_parent_type_different(self):
+        self.parent.corpus = self.corpus1
+        self.child.corpus = self.corpus1
+        self.parent.type = EntityType.Person
+        self.child.type = EntityType.Person
+        with self.assertRaises(ValidationError):
+            self.link.save()
+
+    def test_parent_corpus_different(self):
+        self.parent.corpus = self.corpus2
+        self.child.corpus = self.corpus1
+        self.parent.type = EntityType.Organization
+        self.child.type = EntityType.Person
+        with self.assertRaises(ValidationError):
+            self.link.save()
+
+    def test_child_type_different(self):
+        self.parent.corpus = self.corpus1
+        self.child.corpus = self.corpus1
+        self.parent.type = EntityType.Organization
+        self.child.type = EntityType.Organization
+        with self.assertRaises(ValidationError):
+            self.link.save()
+
+    def test_child_corpus_different(self):
+        self.parent.corpus = self.corpus1
+        self.child.corpus = self.corpus2
+        self.parent.type = EntityType.Organization
+        self.child.type = EntityType.Person
+        with self.assertRaises(ValidationError):
+            self.link.save()
+
+    def test_save_entity_in_metadata(self):
+        self.parent.corpus = self.corpus2
+        element = Element.objects.create(corpus=self.corpus1, type=ElementType.Act, name="element")
+        with self.assertRaises(ValidationError):
+            MetaData.objects.create(
+                name='test 1',
+                type=MetaType.Entity,
+                value='Blah',
+                element=element,
+                entity=self.parent,
+            )

arkindex/project/settings.py

@@ -141,6 +141,8 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django.contrib.postgres',
+    'django_admin_hstore_widget',
 
     # Tools
     'rest_framework',

requirements.txt

@@ -4,6 +4,7 @@ arkindex-common==0.1.0
 boto3==1.9
 certifi==2017.7.27.1
 chardet==3.0.4
+django-admin-hstore-widget==1.0.1
 django-cors-headers==2.4.0
 django-enumfields==1.0.0
 djangorestframework==3.9.2