Merge branch 'telegraf' into 'master'

Add basic cli for telegraf output

See merge request !111

arkindex/documents/management/commands/telegraf.py

+#!/usr/bin/env python3
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from django.utils.text import slugify
+from arkindex.documents.models import Element, ElementType, Transcription, Corpus
+import time
+
+
+class Command(BaseCommand):
+    help = 'Display statistics for a Telegraf agent'
+
+    def handle(self, *args, **options):
+
+        # Total transcriptions
+        self.output(
+            'transcriptions',
+            total=Transcription.objects.count(),
+        )
+
+        # Elements with transcriptions
+        self.output(
+            'pages',
+            total=Element.objects.filter(type=ElementType.Page).count(),
+            with_transcriptions=Transcription.objects.values_list('element', flat=True).distinct().count(),
+        )
+
+        corpus_total = Corpus.objects.filter(elements__type=ElementType.Volume).annotate(Count('elements'))
+        corpus_with_transcriptions = Corpus.objects.raw(
+            """
+            select v.corpus_id as id, count(distinct v.id) as volumes_count
+            from documents_element as e
+            inner join (
+                select distinct element_id
+                from documents_transcription
+            ) as t on (t.element_id = e.id)
+            inner join documents_elementpath as p on (t.element_id=p.element_id)
+            inner join documents_element as v on (v.id=p.path[1])
+            group by v.corpus_id;
+            """,
+        )
+        vol_with_transcriptions_count = {
+            cwt.id: cwt.volumes_count
+            for cwt in corpus_with_transcriptions
+        }
+
+        for corpus in corpus_total:
+            self.output(
+                'corpus',
+                tags={'name': slugify(corpus.name)},
+                total=corpus.elements__count,
+                with_transcriptions=vol_with_transcriptions_count.get(corpus.id, 0),
+            )
+
+    def dict_to_str(self, data):
+        return ','.join([
+            '{}={}'.format(k, v)
+            for (k, v) in data.items()
+        ])
+
+    def output(self, name, tags={}, **values):
+        '''
+        Output on stdout in Influx line protocol
+        https://docs.influxdata.com/influxdb/v1.6/write_protocols/line_protocol_tutorial/
+        '''
+        timestamp = time.time() * 10 ** 9  # in nano seconds
+        tags_str = ',' + self.dict_to_str(tags) if tags else ''
+        values_str = self.dict_to_str(values)
+        line = '{}{} {} {:.0f}'.format(name, tags_str, values_str, timestamp)
+        self.stdout.write(line)

arkindex/documents/tests/test_telegraf.py

+from django.core.management import call_command
+from arkindex.project.tests import FixtureTestCase
+import io
+import re
+
+
+class TestTasks(FixtureTestCase):
+    """Tests for telegraf reporting CLI"""
+
+    def test_command(self):
+
+        output = io.StringIO()
+        call_command('telegraf', stdout=output)
+        self.assertIsNotNone(output)
+
+        # Parse output
+        lines = list(filter(None, output.getvalue().split('\n')))
+        self.assertEqual(len(lines), 3)
+
+        regex = re.compile(r'^(\w+)(,[\w\-_,=]+)? ([\w\-_,=]+) (\d+)$')
+
+        data = {}
+        for line in lines:
+            details = regex.search(line)
+            self.assertIsNotNone(details)
+            name, tags, values, t = details.groups()
+            self.assertTrue(int(t) > 0)
+            data[name] = {
+                k: int(v)
+                for k, v in re.findall(r'(\w+)=(\d+)', values)
+            }
+            data[name]["tags"] = {
+                k: v
+                for k, v in re.findall(r'(\w+)=([\w\-]+)', tags)
+            } if tags else {}
+
+        # Data tests
+        self.assertEqual(data['transcriptions']['total'], 9)
+        self.assertEqual(data['transcriptions']['tags'], {})
+        self.assertEqual(data['pages']['total'], 6)
+        self.assertEqual(data['pages']['with_transcriptions'], 3)
+        self.assertEqual(data['pages']['tags'], {})
+        self.assertEqual(data['corpus']['total'], 2)
+        self.assertEqual(data['corpus']['with_transcriptions'], 1)
+        self.assertDictEqual(data['corpus']['tags'], {'name': 'unit-tests'})