Merge branch 'elements-qs-trash' into 'master'

Generic elements deletion through queryset

Closes #518

See merge request !1067

arkindex/documents/apps.py

@@ -6,4 +6,3 @@ class DocumentsConfig(AppConfig):
 
     def ready(self):
         from arkindex.project import checks  # noqa: F401
-        from arkindex.documents import signals  # noqa: F401

arkindex/documents/managers.py

 import uuid
 from itertools import chain, groupby
 
-from django.db import models
+from django.db import connections, models
 
 from arkindex.project.fields import Unnest
 
 
+class ElementQuerySet(models.QuerySet):
+
+    def trash(self, delete_children=True):
+        """Performant deletion of any element queryset"""
+        from arkindex.documents.models import Element, MetaData, Transcription, TranscriptionEntity, Classification, Selection
+        from arkindex.dataimport.models import DataImport, DataImportElement
+
+        # We only need the Element IDs to join on other models
+        # Disable all ordering as it will not be required for deletions
+        ids = self.order_by().values_list('id')
+
+        # Generate the initial raw SQL query
+        sql, params = ids.query.sql_with_params()
+
+        # Delete all children from the targeted elements
+        # using the same trash() method, iterating on every depth
+        if delete_children is True:
+            # First we need to find out the depths of the top elements' children
+            # The fastest way is to calc all the lengths of children paths
+            # by including the target SQL query, and joining it directly with paths
+            # It's not possible to do that join with Django ORM
+            with connections['default'].cursor() as cursor:
+                cursor.execute("""select min(length), max(length) FROM (
+                        select array_length(p.path, 1) as length
+                        from documents_elementpath as p
+                        inner join
+                        ({}) as input on (array[input.id] && p.path)
+                ) as lengths
+                """.format(sql), params)
+                min_paths, max_paths = cursor.fetchone()
+
+            # Postgres will give us None when no children is found
+            if min_paths is not None and max_paths is not None:
+                # Django increments the path number to match the postgresl 1-index
+                # Remove the lower levels first, then work the way up
+                for i in reversed(range(min_paths - 1, max_paths)):
+                    filters = {
+                        f"paths__path__{i}__in": self,
+                    }
+                    Element.objects.filter(**filters).trash(delete_children=False)
+
+        # TranscriptionEntity is linked through a transcription, so it's a bit harder to join
+        TranscriptionEntity.objects.filter(transcription__element_id__in=ids).prefetch_related('transcription').delete()
+
+        # We need to use raw deletion instead of the smarter delete from Django
+        # to avoid loading in RAM the transcription ids to check that transcription entity is deleted cleanly
+        Transcription.objects.filter(element_id__in=ids)._raw_delete(using='default')
+
+        # Simple direct dependencies to remove, through a Foreign Key
+        for cls in (MetaData, Classification, Selection, DataImportElement):
+            cls.objects.filter(element_id__in=ids).delete()
+
+        # Set elements directly on dataimports to None
+        DataImport.objects.filter(element_id__in=ids).update(element_id=None)
+
+        # This is where it gets really ugly: in order to delete all the children
+        # without losing their reference, we need to inject the generated SQL query
+        # directly into an SQL DELETE statement for paths
+        # Once paths are deleted, we can finally delete the targeted elements
+        with connections['default'].cursor() as cursor:
+            cursor.execute("""
+            WITH element_ids (id) AS (
+                DELETE FROM documents_elementpath
+                WHERE element_id IN ({})
+                RETURNING element_id
+            )
+            DELETE FROM documents_element element
+            USING element_ids
+            WHERE element.id = element_ids.id
+            """.format(sql), params)
+
+        # Finally, delete top elements
+        self.delete()
+
+
 class ElementManager(models.Manager):
     """Model manager for elements"""
 
+    def get_queryset(self):
+        return ElementQuerySet(self.model, using=self._db)
+
     def get_ascending(self, child_id, recursive=True):
         """
         Get all parent elements for a specific element ID.

arkindex/documents/signals.py

-from django.db.models.signals import pre_delete
-from django.dispatch import receiver
-
-from arkindex.documents.models import Element
-
-
-@receiver(pre_delete, sender=Element)
-def pre_delete_handler(sender, instance, *args, **kwargs):
-    raise NotImplementedError('Deleting elements from a QuerySet is not supported. Please call Element.delete on each element.')

arkindex/documents/tasks.py

@@ -4,7 +4,6 @@ from typing import Optional
 
 from django.db.models import Q
 from django.db.models.deletion import Collector
-from django.db.models.signals import pre_delete
 from django_rq import job
 
 from arkindex.dataimport.models import DataImport, DataImportElement, WorkerRun
@@ -21,8 +20,6 @@ from arkindex.documents.models import (
     Transcription,
     TranscriptionEntity,
 )
-from arkindex.documents.signals import pre_delete_handler
-from arkindex.project.tools import disconnect_signal
 
 logger = logging.getLogger(__name__)
 
@@ -191,9 +188,8 @@ def corpus_delete(corpus_id: str) -> None:
         Corpus.objects.filter(id=corpus_id),
     ]
 
-    with disconnect_signal(pre_delete, sender=Element, receiver=pre_delete_handler):
-        for queryset in querysets:
-            deleted_count = queryset._raw_delete(using='default')
-            logger.info(f'Deleted {deleted_count} {queryset.model.__name__}')
+    for queryset in querysets:
+        deleted_count = queryset._raw_delete(using='default')
+        logger.info(f'Deleted {deleted_count} {queryset.model.__name__}')
 
     logger.info(f'Deleted corpus {corpus_id}')

arkindex/documents/tests/test_destroy_elements.py

@@ -3,6 +3,7 @@ from rest_framework import status
 
 from arkindex.documents.models import Corpus, Element
 from arkindex.project.tests import FixtureAPITestCase
+from arkindex.project.tools import build_tree
 
 
 class TestDestroyElements(FixtureAPITestCase):
@@ -57,3 +58,99 @@ class TestDestroyElements(FixtureAPITestCase):
 
         with self.assertRaises(Element.DoesNotExist):
             self.vol.refresh_from_db()
+
+    def test_element_trash_children(self):
+        # Check this volume has children
+        children = Element.objects.get_descending(self.vol.id)
+        ids = list(children.values_list('id', flat=True))
+        self.assertEqual(len(ids), 24)
+        self.assertEqual(len(Element.objects.exclude(id__in=ids)), 5)
+
+        with self.assertExactQueries('element_trash_children.sql', params={'id': str(self.vol.id)}):
+            Element.objects.filter(id=self.vol.id).trash()
+
+        with self.assertRaises(Element.DoesNotExist):
+            self.vol.refresh_from_db()
+
+        self.assertFalse(Element.objects.filter(id__in=ids).exists())
+        # Excluding the children, only the volume should have been deleted
+        self.assertEqual(len(Element.objects.exclude(id__in=ids)), 4)
+
+    def test_element_trash_without_children(self):
+        children = Element.objects.get_descending(self.vol.id)
+        ids = list(children.values_list('id', flat=True))
+        self.assertEqual(len(ids), 24)
+
+        with self.assertExactQueries('element_trash_no_children.sql', params={'id': str(self.vol.id)}):
+            Element.objects.filter(id=self.vol.id).trash(delete_children=False)
+
+        with self.assertRaises(Element.DoesNotExist):
+            self.vol.refresh_from_db()
+
+        # No children were deleted
+        self.assertEqual(Element.objects.filter(id__in=ids).count(), 24)
+
+    def test_element_trash_no_order_by(self):
+        children = Element.objects.get_descending(self.vol.id)
+        ids = list(children.values_list('id', flat=True))
+        self.assertEqual(len(ids), 24)
+
+        with self.assertExactQueries('element_trash_children.sql', params={'id': str(self.vol.id)}):
+            Element.objects.filter(id=self.vol.id).order_by('name').trash()
+
+        with self.assertRaises(Element.DoesNotExist):
+            self.vol.refresh_from_db()
+
+        self.assertFalse(Element.objects.filter(id__in=ids).exists())
+
+    def test_element_trash_related_filter_bug(self):
+        """
+        This test should fail when a bug in ElementQuerySet.trash is fixed:
+        Filtering by metadata, classification, selection, DataImportElement, or DataImport
+        can cause elements to not actually be deleted as the related tables are deleted first.
+        For example, deleting all elements with classifications will cause all classifications
+        to be deleted, but not elements, because the QuerySet is re-evaluated
+        and no elements will have classifications.
+        """
+        ml_class = self.corpus.ml_classes.create(name='Krabby')
+        page1r, page1v, page2r = Element.objects.get_descending(self.vol.id).filter(type__slug='page')
+        page1r.classifications.create(ml_class=ml_class, confidence=0.42)
+        page1v.classifications.create(ml_class=ml_class, confidence=0.42)
+        page2r.classifications.create(ml_class=ml_class, confidence=0.42)
+
+        queryset = Element.objects.filter(classifications__isnull=False)
+
+        # Ensure the pages are all in this queryset, and therefore should be deleted
+        self.assertEqual(queryset.filter(id__in=[page1r.id, page1v.id, page2r.id]).count(), 3)
+
+        with self.assertExactQueries('element_trash_ml_class.sql'):
+            Element.objects.filter(classifications__isnull=False).trash()
+
+        # Pages did not get deleted
+        page1r.refresh_from_db()
+        page1v.refresh_from_db()
+        page2r.refresh_from_db()
+
+        # Classifications have been deleted
+        self.assertFalse(ml_class.classifications.exists())
+
+    def test_element_trash_deep(self):
+        """
+        Tests the repetition of path deletions with deep hierarchies
+        Builds a tree of elements A→B→C→D→E and deletes A with its children
+        """
+        elements = build_tree(
+            {
+                'B': 'A',
+                'C': 'B',
+                'D': 'C',
+                'E': 'D',
+            },
+            type=self.corpus.types.create(slug='houndour', display_name='Houndour', folder=True),
+            corpus=self.corpus,
+        )
+
+        with self.assertExactQueries('element_trash_deep.sql', params={'id': str(elements['A'].id)}):
+            Element.objects.filter(id=elements['A'].id).trash()
+
+        self.assertFalse(Element.objects.filter(id__in=[e.id for e in elements.values()]).exists())

arkindex/sql_validation/corpus_delete.sql

@@ -6,12 +6,12 @@ SELECT "documents_corpus"."created",
        "documents_corpus"."repository_id",
        "documents_corpus"."public"
 FROM "documents_corpus"
-WHERE "documents_corpus"."id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid
+WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
 LIMIT 21;
 
 DELETE
 FROM "documents_allowedmetadata"
-WHERE "documents_allowedmetadata"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "documents_allowedmetadata"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "dataimport_workerrun"
@@ -19,7 +19,7 @@ WHERE "dataimport_workerrun"."id" IN
         (SELECT U0."id"
          FROM "dataimport_workerrun" U0
          INNER JOIN "dataimport_dataimport" U1 ON (U0."dataimport_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "dataimport_dataimportelement"
@@ -27,7 +27,7 @@ WHERE "dataimport_dataimportelement"."id" IN
         (SELECT U0."id"
          FROM "dataimport_dataimportelement" U0
          INNER JOIN "dataimport_dataimport" U1 ON (U0."dataimport_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "dataimport_dataimportelement"
@@ -35,7 +35,7 @@ WHERE "dataimport_dataimportelement"."id" IN
         (SELECT U0."id"
          FROM "dataimport_dataimportelement" U0
          INNER JOIN "documents_element" U1 ON (U0."element_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "dataimport_dataimport_files"
@@ -43,7 +43,7 @@ WHERE "dataimport_dataimport_files"."id" IN
         (SELECT U0."id"
          FROM "dataimport_dataimport_files" U0
          INNER JOIN "dataimport_dataimport" U1 ON (U0."dataimport_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "dataimport_dataimport_files"
@@ -51,15 +51,15 @@ WHERE "dataimport_dataimport_files"."id" IN
         (SELECT U0."id"
          FROM "dataimport_dataimport_files" U0
          INNER JOIN "dataimport_datafile" U1 ON (U0."datafile_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "dataimport_dataimport"
-WHERE "dataimport_dataimport"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "dataimport_dataimport"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "dataimport_datafile"
-WHERE "dataimport_datafile"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "dataimport_datafile"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "documents_metadata"
@@ -67,7 +67,7 @@ WHERE "documents_metadata"."id" IN
         (SELECT U0."id"
          FROM "documents_metadata" U0
          INNER JOIN "documents_element" U1 ON (U0."element_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_entitylink"
@@ -75,11 +75,11 @@ WHERE "documents_entitylink"."id" IN
         (SELECT U0."id"
          FROM "documents_entitylink" U0
          INNER JOIN "documents_entityrole" U1 ON (U0."role_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_entityrole"
-WHERE "documents_entityrole"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "documents_entityrole"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "documents_transcriptionentity"
@@ -87,7 +87,7 @@ WHERE "documents_transcriptionentity"."id" IN
         (SELECT U0."id"
          FROM "documents_transcriptionentity" U0
          INNER JOIN "documents_entity" U1 ON (U0."entity_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_transcriptionentity"
@@ -96,11 +96,11 @@ WHERE "documents_transcriptionentity"."id" IN
          FROM "documents_transcriptionentity" U0
          INNER JOIN "documents_transcription" U1 ON (U0."transcription_id" = U1."id")
          INNER JOIN "documents_element" U2 ON (U1."element_id" = U2."id")
-         WHERE U2."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U2."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_entity"
-WHERE "documents_entity"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "documents_entity"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "documents_classification"
@@ -108,11 +108,11 @@ WHERE "documents_classification"."id" IN
         (SELECT U0."id"
          FROM "documents_classification" U0
          INNER JOIN "documents_element" U1 ON (U0."element_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_mlclass"
-WHERE "documents_mlclass"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "documents_mlclass"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "documents_transcription"
@@ -120,7 +120,7 @@ WHERE "documents_transcription"."id" IN
         (SELECT U0."id"
          FROM "documents_transcription" U0
          INNER JOIN "documents_element" U1 ON (U0."element_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_elementpath"
@@ -128,7 +128,7 @@ WHERE "documents_elementpath"."id" IN
         (SELECT U0."id"
          FROM "documents_elementpath" U0
          INNER JOIN "documents_element" U1 ON (U0."element_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_selection"
@@ -136,20 +136,20 @@ WHERE "documents_selection"."id" IN
         (SELECT U0."id"
          FROM "documents_selection" U0
          INNER JOIN "documents_element" U1 ON (U0."element_id" = U1."id")
-         WHERE U1."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid);
+         WHERE U1."corpus_id" = '{corpus_id}'::uuid);
 
 DELETE
 FROM "documents_element"
-WHERE "documents_element"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "documents_element"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "documents_elementtype"
-WHERE "documents_elementtype"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "documents_elementtype"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "users_corpusright"
-WHERE "users_corpusright"."corpus_id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid;
+WHERE "users_corpusright"."corpus_id" = '{corpus_id}'::uuid;
 
 DELETE
 FROM "documents_corpus"
-WHERE "documents_corpus"."id" = '78d45b37-36d1-4576-8bb2-8e5844f31feb'::uuid
\ No newline at end of file
+WHERE "documents_corpus"."id" = '{corpus_id}'::uuid

arkindex/sql_validation/element_trash_children.sql

+
+select min(length),
+       max(length)
+FROM
+    (select array_length(p.path, 1) as length
+     from documents_elementpath as p
+     inner join
+         (SELECT "documents_element"."id"
+          FROM "documents_element"
+          WHERE "documents_element"."id" = '{id}'::uuid) as input on (array[input.id] && p.path)) as lengths ;
+
+DELETE
+FROM "documents_transcriptionentity"
+WHERE "documents_transcriptionentity"."id" IN
+        (SELECT W0."id"
+         FROM "documents_transcriptionentity" W0
+         INNER JOIN "documents_transcription" W1 ON (W0."transcription_id" = W1."id")
+         WHERE W1."element_id" IN
+                 (SELECT V0."id"
+                  FROM "documents_element" V0
+                  INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+                  WHERE V1."path"[2] IN
+                          (SELECT U0."id"
+                           FROM "documents_element" U0
+                           WHERE U0."id" = '{id}'::uuid)));
+
+DELETE
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+UPDATE "dataimport_dataimport"
+SET "element_id" = NULL
+WHERE "dataimport_dataimport"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+WITH element_ids (id) AS
+    (DELETE
+     FROM documents_elementpath
+     WHERE element_id IN
+             (SELECT "documents_element"."id"
+              FROM "documents_element"
+              INNER JOIN "documents_elementpath" ON ("documents_element"."id" = "documents_elementpath"."element_id")
+              WHERE "documents_elementpath"."path"[2] IN
+                      (SELECT U0."id"
+                       FROM "documents_element" U0
+                       WHERE U0."id" = '{id}'::uuid)) RETURNING element_id)
+DELETE
+FROM documents_element element USING element_ids
+WHERE element.id = element_ids.id ;
+
+SELECT "documents_element"."id",
+       "documents_element"."created",
+       "documents_element"."updated",
+       "documents_element"."corpus_id",
+       "documents_element"."type_id",
+       "documents_element"."name",
+       "documents_element"."zone_id",
+       "documents_element"."source_id",
+       "documents_element"."worker_version_id"
+FROM "documents_element"
+INNER JOIN "documents_elementpath" ON ("documents_element"."id" = "documents_elementpath"."element_id")
+WHERE "documents_elementpath"."path"[2] IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_transcriptionentity"
+WHERE "documents_transcriptionentity"."id" IN
+        (SELECT W0."id"
+         FROM "documents_transcriptionentity" W0
+         INNER JOIN "documents_transcription" W1 ON (W0."transcription_id" = W1."id")
+         WHERE W1."element_id" IN
+                 (SELECT V0."id"
+                  FROM "documents_element" V0
+                  INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+                  WHERE V1."path"[1] IN
+                          (SELECT U0."id"
+                           FROM "documents_element" U0
+                           WHERE U0."id" = '{id}'::uuid)));
+
+DELETE
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[1] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[1] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[1] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[1] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[1] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+UPDATE "dataimport_dataimport"
+SET "element_id" = NULL
+WHERE "dataimport_dataimport"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[1] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+WITH element_ids (id) AS
+    (DELETE
+     FROM documents_elementpath
+     WHERE element_id IN
+             (SELECT "documents_element"."id"
+              FROM "documents_element"
+              INNER JOIN "documents_elementpath" ON ("documents_element"."id" = "documents_elementpath"."element_id")
+              WHERE "documents_elementpath"."path"[1] IN
+                      (SELECT U0."id"
+                       FROM "documents_element" U0
+                       WHERE U0."id" = '{id}'::uuid)) RETURNING element_id)
+DELETE
+FROM documents_element element USING element_ids
+WHERE element.id = element_ids.id ;
+
+SELECT "documents_element"."id",
+       "documents_element"."created",
+       "documents_element"."updated",
+       "documents_element"."corpus_id",
+       "documents_element"."type_id",
+       "documents_element"."name",
+       "documents_element"."zone_id",
+       "documents_element"."source_id",
+       "documents_element"."worker_version_id"
+FROM "documents_element"
+INNER JOIN "documents_elementpath" ON ("documents_element"."id" = "documents_elementpath"."element_id")
+WHERE "documents_elementpath"."path"[1] IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_transcriptionentity"
+WHERE "documents_transcriptionentity"."id" IN
+        (SELECT V0."id"
+         FROM "documents_transcriptionentity" V0
+         INNER JOIN "documents_transcription" V1 ON (V0."transcription_id" = V1."id")
+         WHERE V1."element_id" IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+UPDATE "dataimport_dataimport"
+SET "element_id" = NULL
+WHERE "dataimport_dataimport"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+WITH element_ids (id) AS
+    (DELETE
+     FROM documents_elementpath
+     WHERE element_id IN
+             (SELECT "documents_element"."id"
+              FROM "documents_element"
+              WHERE "documents_element"."id" = '{id}'::uuid) RETURNING element_id)
+DELETE
+FROM documents_element element USING element_ids
+WHERE element.id = element_ids.id ;
+
+SELECT "documents_element"."id",
+       "documents_element"."created",
+       "documents_element"."updated",
+       "documents_element"."corpus_id",
+       "documents_element"."type_id",
+       "documents_element"."name",
+       "documents_element"."zone_id",
+       "documents_element"."source_id",
+       "documents_element"."worker_version_id"
+FROM "documents_element"
+WHERE "documents_element"."id" = '{id}'::uuid;
+
+SELECT "documents_transcription"."id"
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN ('{id}'::uuid);
+
+SELECT "dataimport_dataimport"."id"
+FROM "dataimport_dataimport"
+WHERE "dataimport_dataimport"."element_id" IN ('{id}'::uuid)
+ORDER BY "dataimport_dataimport"."corpus_id" ASC,
+         "dataimport_dataimport"."created" DESC;
+
+DELETE
+FROM "documents_elementpath"
+WHERE "documents_elementpath"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_element"
+WHERE "documents_element"."id" IN ('{id}'::uuid)

arkindex/sql_validation/element_trash_ml_class.sql

+select min(length),
+       max(length)
+FROM
+    (select array_length(p.path, 1) as length
+     from documents_elementpath as p
+     inner join
+         (SELECT "documents_element"."id"
+          FROM "documents_element"
+          INNER JOIN "documents_classification" ON ("documents_element"."id" = "documents_classification"."element_id")
+          WHERE "documents_classification"."id" IS NOT NULL) as input on (array[input.id] && p.path)) as lengths ;
+
+DELETE
+FROM "documents_transcriptionentity"
+WHERE "documents_transcriptionentity"."id" IN
+        (SELECT W0."id"
+         FROM "documents_transcriptionentity" W0
+         INNER JOIN "documents_transcription" W1 ON (W0."transcription_id" = W1."id")
+         WHERE W1."element_id" IN
+                 (SELECT V0."id"
+                  FROM "documents_element" V0
+                  INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+                  WHERE V1."path"[2] IN
+                          (SELECT U0."id"
+                           FROM "documents_element" U0
+                           INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                           WHERE U1."id" IS NOT NULL)));
+
+DELETE
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+UPDATE "dataimport_dataimport"
+SET "element_id" = NULL
+WHERE "dataimport_dataimport"."element_id" IN
+        (SELECT V0."id"
+         FROM "documents_element" V0
+         INNER JOIN "documents_elementpath" V1 ON (V0."id" = V1."element_id")
+         WHERE V1."path"[2] IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+WITH element_ids (id) AS
+    (DELETE
+     FROM documents_elementpath
+     WHERE element_id IN
+             (SELECT "documents_element"."id"
+              FROM "documents_element"
+              INNER JOIN "documents_elementpath" ON ("documents_element"."id" = "documents_elementpath"."element_id")
+              WHERE "documents_elementpath"."path"[2] IN
+                      (SELECT U0."id"
+                       FROM "documents_element" U0
+                       INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                       WHERE U1."id" IS NOT NULL)) RETURNING element_id)
+DELETE
+FROM documents_element element USING element_ids
+WHERE element.id = element_ids.id ;
+
+SELECT "documents_element"."id",
+       "documents_element"."created",
+       "documents_element"."updated",
+       "documents_element"."corpus_id",
+       "documents_element"."type_id",
+       "documents_element"."name",
+       "documents_element"."zone_id",
+       "documents_element"."source_id",
+       "documents_element"."worker_version_id"
+FROM "documents_element"
+INNER JOIN "documents_elementpath" ON ("documents_element"."id" = "documents_elementpath"."element_id")
+WHERE "documents_elementpath"."path"[2] IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+DELETE
+FROM "documents_transcriptionentity"
+WHERE "documents_transcriptionentity"."id" IN
+        (SELECT V0."id"
+         FROM "documents_transcriptionentity" V0
+         INNER JOIN "documents_transcription" V1 ON (V0."transcription_id" = V1."id")
+         WHERE V1."element_id" IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+                  WHERE U1."id" IS NOT NULL));
+
+DELETE
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+UPDATE "dataimport_dataimport"
+SET "element_id" = NULL
+WHERE "dataimport_dataimport"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         INNER JOIN "documents_classification" U1 ON (U0."id" = U1."element_id")
+         WHERE U1."id" IS NOT NULL);
+
+WITH element_ids (id) AS
+    (DELETE
+     FROM documents_elementpath
+     WHERE element_id IN
+             (SELECT "documents_element"."id"
+              FROM "documents_element"
+              INNER JOIN "documents_classification" ON ("documents_element"."id" = "documents_classification"."element_id")
+              WHERE "documents_classification"."id" IS NOT NULL) RETURNING element_id)
+DELETE
+FROM documents_element element USING element_ids
+WHERE element.id = element_ids.id ;
+
+SELECT "documents_element"."id",
+       "documents_element"."created",
+       "documents_element"."updated",
+       "documents_element"."corpus_id",
+       "documents_element"."type_id",
+       "documents_element"."name",
+       "documents_element"."zone_id",
+       "documents_element"."source_id",
+       "documents_element"."worker_version_id"
+FROM "documents_element"
+INNER JOIN "documents_classification" ON ("documents_element"."id" = "documents_classification"."element_id")
+WHERE "documents_classification"."id" IS NOT NULL
\ No newline at end of file

arkindex/sql_validation/element_trash_no_children.sql

+DELETE
+FROM "documents_transcriptionentity"
+WHERE "documents_transcriptionentity"."id" IN
+        (SELECT V0."id"
+         FROM "documents_transcriptionentity" V0
+         INNER JOIN "documents_transcription" V1 ON (V0."transcription_id" = V1."id")
+         WHERE V1."element_id" IN
+                 (SELECT U0."id"
+                  FROM "documents_element" U0
+                  WHERE U0."id" = '{id}'::uuid));
+
+DELETE
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+UPDATE "dataimport_dataimport"
+SET "element_id" = NULL
+WHERE "dataimport_dataimport"."element_id" IN
+        (SELECT U0."id"
+         FROM "documents_element" U0
+         WHERE U0."id" = '{id}'::uuid);
+
+WITH element_ids (id) AS
+    (DELETE
+     FROM documents_elementpath
+     WHERE element_id IN
+             (SELECT "documents_element"."id"
+              FROM "documents_element"
+              WHERE "documents_element"."id" = '{id}'::uuid) RETURNING element_id)
+DELETE
+FROM documents_element element USING element_ids
+WHERE element.id = element_ids.id ;
+
+SELECT "documents_element"."id",
+       "documents_element"."created",
+       "documents_element"."updated",
+       "documents_element"."corpus_id",
+       "documents_element"."type_id",
+       "documents_element"."name",
+       "documents_element"."zone_id",
+       "documents_element"."source_id",
+       "documents_element"."worker_version_id"
+FROM "documents_element"
+WHERE "documents_element"."id" = '{id}'::uuid;
+
+SELECT "documents_transcription"."id"
+FROM "documents_transcription"
+WHERE "documents_transcription"."element_id" IN ('{id}'::uuid);
+
+SELECT "dataimport_dataimport"."id"
+FROM "dataimport_dataimport"
+WHERE "dataimport_dataimport"."element_id" IN ('{id}'::uuid)
+ORDER BY "dataimport_dataimport"."corpus_id" ASC,
+         "dataimport_dataimport"."created" DESC;
+
+DELETE
+FROM "documents_elementpath"
+WHERE "documents_elementpath"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_classification"
+WHERE "documents_classification"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_metadata"
+WHERE "documents_metadata"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_selection"
+WHERE "documents_selection"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "dataimport_dataimportelement"
+WHERE "dataimport_dataimportelement"."element_id" IN ('{id}'::uuid);
+
+DELETE
+FROM "documents_element"
+WHERE "documents_element"."id" IN ('{id}'::uuid)