Skip to content
Snippets Groups Projects
Commit e71a3763 authored by Bastien Abadie's avatar Bastien Abadie Committed by Erwan Rouchet
Browse files

Optimize ListElementLinks SQL queries by using prefetch and more explicit queries

parent 2dd45ed3
No related branches found
No related tags found
1 merge request!1187Optimize ListElementLinks SQL queries by using prefetch and more explicit queries
Hide whitespace changes
Inline Side-by-side
arkindex/documents/api/entities.py
+ 17
− 3
View file @ e71a3763
......@@ -322,6 +322,20 @@ class ElementLinks(ListAPIView):
element = Element.objects.get(id=self.kwargs['pk'])
except Element.DoesNotExist:
raise NotFound
entities = Entity.objects.filter(Q(metadatas__element_id=element.id) | Q(transcriptions__element_id=element.id))
return EntityLink.objects.filter(parent__in=entities, child__in=entities) \
.prefetch_related('role', 'child', 'parent').order_by('parent__name')
# Load entities linked by transcriptions
entities_tr = Entity.objects.filter(transcriptions__element_id=element.id).prefetch_related('transcriptions')
# Load entities linked by metadatas
entities_meta = Entity.objects.filter(metadatas__element_id=element.id).prefetch_related('metadatas')
# Now load all links belonging to those entities
# It's several times faster to combine the queries in the final one
# than combining them at the lower level (entities is slower than entities_tr + entities_meta)
# We need to support cross references between transcriptions & metadata entities
return EntityLink.objects.filter(
Q(parent__in=entities_tr, child__in=entities_tr)
| Q(parent__in=entities_tr, child__in=entities_meta)
| Q(parent__in=entities_meta, child__in=entities_tr)
| Q(parent__in=entities_meta, child__in=entities_meta)
).select_related('role', 'child', 'parent').order_by('parent__name')
arkindex/documents/tests/test_entities_api.py
+ 4
− 2
View file @ e71a3763
......@@ -861,7 +861,8 @@ class TestEntitiesAPI(FixtureAPITestCase):
def test_list_element_links(self):
link = EntityLink.objects.create(parent=self.entity, child=self.entity_bis, role=self.role)
response = self.client.get(reverse('api:element-links', kwargs={'pk': str(self.element.id)}))
with self.assertExactQueries('element_links.sql', params={'element_id': self.element.id}):
response = self.client.get(reverse('api:element-links', kwargs={'pk': str(self.element.id)}))
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json()
self.assertListEqual(
......@@ -898,7 +899,8 @@ class TestEntitiesAPI(FixtureAPITestCase):
def test_list_element_links_not_found(self):
self.assertFalse(Element.objects.filter(id='aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa').exists())
response = self.client.get(reverse('api:element-links', kwargs={'pk': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'}))
with self.assertExactQueries('element_links_not_found.sql', params={'element_id': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'}):
response = self.client.get(reverse('api:element-links', kwargs={'pk': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
@patch('arkindex.project.triggers.tasks.reindex_start.delay')
......
arkindex/sql_validation/element_links.sql 0 → 100644
+ 135
− 0
View file @ e71a3763
SELECT "documents_element"."id",
"documents_element"."created",
"documents_element"."updated",
"documents_element"."corpus_id",
"documents_element"."type_id",
"documents_element"."name",
"documents_element"."zone_id",
"documents_element"."worker_version_id"
FROM "documents_element"
WHERE "documents_element"."id" = '{element_id}'::uuid
LIMIT 21;
SELECT COUNT(*) AS "__count"
FROM "documents_entitylink"
WHERE (("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid))
OR ("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid))
OR ("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid))
OR ("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid)));
SELECT "documents_entitylink"."id",
"documents_entitylink"."parent_id",
"documents_entitylink"."child_id",
"documents_entitylink"."role_id",
T3."id",
T3."name",
T3."type",
T3."corpus_id",
T3."metas",
T3."validated",
T3."moderator_id",
T3."worker_version_id",
"documents_entity"."id",
"documents_entity"."name",
"documents_entity"."type",
"documents_entity"."corpus_id",
"documents_entity"."metas",
"documents_entity"."validated",
"documents_entity"."moderator_id",
"documents_entity"."worker_version_id",
"documents_entityrole"."id",
"documents_entityrole"."parent_name",
"documents_entityrole"."child_name",
"documents_entityrole"."parent_type",
"documents_entityrole"."child_type",
"documents_entityrole"."corpus_id"
FROM "documents_entitylink"
INNER JOIN "documents_entity" ON ("documents_entitylink"."child_id" = "documents_entity"."id")
INNER JOIN "documents_entity" T3 ON ("documents_entitylink"."parent_id" = T3."id")
INNER JOIN "documents_entityrole" ON ("documents_entitylink"."role_id" = "documents_entityrole"."id")
WHERE (("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid))
OR ("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid))
OR ("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_transcriptionentity" U1 ON (U0."id" = U1."entity_id")
INNER JOIN "documents_transcription" U2 ON (U1."transcription_id" = U2."id")
WHERE U2."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid))
OR ("documents_entitylink"."child_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid)
AND "documents_entitylink"."parent_id" IN
(SELECT U0."id"
FROM "documents_entity" U0
INNER JOIN "documents_metadata" U1 ON (U0."id" = U1."entity_id")
WHERE U1."element_id" = '{element_id}'::uuid)))
ORDER BY T3."name" ASC
LIMIT 1
arkindex/sql_validation/element_links_not_found.sql 0 → 100644
+ 11
− 0
View file @ e71a3763
SELECT "documents_element"."id",
"documents_element"."created",
"documents_element"."updated",
"documents_element"."corpus_id",
"documents_element"."type_id",
"documents_element"."name",
"documents_element"."zone_id",
"documents_element"."worker_version_id"
FROM "documents_element"
WHERE "documents_element"."id" = '{element_id}'::uuid
LIMIT 21
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment