diff --git a/arkindex/documents/management/commands/telegraf.py b/arkindex/documents/management/commands/telegraf.py deleted file mode 100644 index 53907b9f126109bfa8f8061e410d18e2c8c3068e..0000000000000000000000000000000000000000 --- a/arkindex/documents/management/commands/telegraf.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -import time -from urllib.parse import urljoin - -import requests -from django.conf import settings -from django.core.management.base import BaseCommand -from django.db.models import Count - -from arkindex.images.models import Image, ImageServer -from arkindex.project.aws import S3FileStatus - - -class Command(BaseCommand): - help = 'Display statistics for a Telegraf agent' - - def add_arguments(self, parser): - parser.add_argument( - '--influxdb', - help='Post statistics to the InfluxDB API at a given URL', - nargs='?', - const=settings.INFLUXDB_API_URL, - ) - - def handle(self, *args, **options): - self.options = options - - # Image server statistics, in three SQL queries - checked_counts = dict( - Image.objects - .filter(status=S3FileStatus.Checked) - .values_list('server') - .annotate(Count('server')) - ) - error_counts = dict( - Image.objects - .filter(status=S3FileStatus.Error) - .values_list('server') - .annotate(Count('server')) - ) - for server in ImageServer.objects.annotate(img_total=Count('images')): - self.output( - 'images', - tags={'imageserver': server.grafana_tag}, - total=server.img_total, - checked=checked_counts.get(server.id, 0), - errors=error_counts.get(server.id, 0), - ) - - def dict_to_str(self, data): - return ','.join([ - '{}={}'.format(k, str(v).replace(' ', '_')) - for (k, v) in data.items() - ]) - - def output(self, name, tags={}, **values): - ''' - Output on stdout in Influx line protocol - https://docs.influxdata.com/influxdb/v1.6/write_protocols/line_protocol_tutorial/ - ''' - timestamp = time.time() * 10 ** 9 # in nano seconds - tags_str = ',' + self.dict_to_str(tags) if tags else '' - values_str = self.dict_to_str(values) - line = '{}{} {} {:.0f}'.format(name, tags_str, values_str, timestamp) - self.stdout.write(line) - - if not self.options.get('influxdb'): - return - resp = requests.post(urljoin(self.options['influxdb'], 'write'), params={'db': 'arkindex'}, data=line) - resp.raise_for_status() diff --git a/arkindex/documents/tests/commands/test_telegraf.py b/arkindex/documents/tests/commands/test_telegraf.py deleted file mode 100644 index f5ea443597aae02b94e695ee221a6d5b252f7a09..0000000000000000000000000000000000000000 --- a/arkindex/documents/tests/commands/test_telegraf.py +++ /dev/null @@ -1,87 +0,0 @@ -import io -import re -from unittest.mock import patch - -from django.core.management import call_command - -from arkindex.project.tests import FixtureTestCase - - -class TestTelegraf(FixtureTestCase): - """Tests for telegraf reporting CLI""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.influx_re = re.compile(r'^(\w+)(,[\w\-_,=]+)? ([\w\-_,=]+) (\d+)$') - cls.influx_measurements_re = re.compile(r'(\w+)=(\d+)') - cls.influx_tags_re = re.compile(r'(\w+)=([\w\-]+)') - cls.expected_output = { - 'images': { - 'total': 6, - 'checked': 0, - 'errors': 0, - 'tags': {'imageserver': 'Test_Server_1'}, - } - } - - def _parse_influx_lines(self, lines): - data = {} - for line in lines: - details = self.influx_re.search(line) - self.assertIsNotNone(details) - name, tags, values, t = details.groups() - self.assertTrue(int(t) > 0) - data[name] = { - k: int(v) - for k, v in self.influx_measurements_re.findall(values) - } - data[name]["tags"] = { - k: v - for k, v in self.influx_tags_re.findall(tags) - } if tags else {} - return data - - def test_command(self): - - output = io.StringIO() - call_command('telegraf', stdout=output) - self.assertIsNotNone(output) - - lines = list(filter(None, output.getvalue().split('\n'))) - self.assertEqual(len(lines), 1) - - self.assertDictEqual( - self._parse_influx_lines(lines), - self.expected_output, - ) - - @patch('arkindex.documents.management.commands.telegraf.requests') - def test_post_influxdb(self, requests_mock): - output = io.StringIO() - call_command( - 'telegraf', - influxdb='http://somewhere:8086/write', - stdout=output, - ) - self.assertIsNotNone(output) - lines = list(filter(None, output.getvalue().split('\n'))) - - self.assertDictEqual( - self._parse_influx_lines(lines), - self.expected_output, - ) - - self.assertEqual(requests_mock.post.call_count, 1) - args, kwargs = zip(*requests_mock.post.call_args_list) - - # All requests are to the same URL - self.assertSetEqual(set(args), {('http://somewhere:8086/write', )}) - - self.assertSequenceEqual(kwargs, [ - {'params': {'db': 'arkindex'}, 'data': line} - for line in lines - ]) - - # Raise exceptions on each POST request - self.assertEqual(requests_mock.post().raise_for_status.call_count, 1) diff --git a/arkindex/images/models.py b/arkindex/images/models.py index a3b7ca6a9cf312aa0837bd5619d41eff9513a4b5..184deaa3d667f00eaff5190d2db64c4ec128238d 100644 --- a/arkindex/images/models.py +++ b/arkindex/images/models.py @@ -153,10 +153,6 @@ class ImageServer(models.Model): ) return new_server - @property - def grafana_tag(self): - return '{}_{}'.format(self.display_name, self.id) - class Image(S3FileMixin, IndexableModel): """ diff --git a/arkindex/project/config.py b/arkindex/project/config.py index 7c997dafa6d29816f39b612d9128931056390f70..51ca14400493644ed4b13f92f3e901c31d176361 100644 --- a/arkindex/project/config.py +++ b/arkindex/project/config.py @@ -123,9 +123,6 @@ def get_settings_parser(base_dir): solr_parser = parser.add_subparser('solr', default={}) solr_parser.add_option('api_url', type=str, default='http://localhost:8983/solr/') - influxdb_parser = parser.add_subparser('influxdb', default={}) - influxdb_parser.add_option('api_url', type=str, default='http://localhost:8086/') - doorbell_parser = parser.add_subparser('doorbell', default={}) doorbell_parser.add_option('id', type=str, default=None) doorbell_parser.add_option('appkey', type=str, default=None) diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index 8d83fae1e4ef7ccffacdd33214b932dfaee1168e..9b54398f0f6614a0566c00d6f18d81c3b5e685e1 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -288,9 +288,6 @@ SOLR_PAGINATION_SIZE = 20 SEARCH_FILTER_MAX_TERMS = 10 -# InfluxDB API root -INFLUXDB_API_URL = conf['influxdb']['api_url'] - # Use SSL proxy SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml index a766faa9aebb8ecb0c9e55491543d96397706f2e..bfd72e6de32418bfa65d8b02a2ffb794126419e3 100644 --- a/arkindex/project/tests/config_samples/defaults.yaml +++ b/arkindex/project/tests/config_samples/defaults.yaml @@ -43,8 +43,6 @@ gitlab: app_id: null app_secret: null imports_worker_version: null -influxdb: - api_url: http://localhost:8086/ ingest: access_key_id: null endpoint: null diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml index 96a7ca9d1b7fbcd83be261f5efe2db442a89d432..5d910c8eb4143a678227ebb4dc7b716131439d84 100644 --- a/arkindex/project/tests/config_samples/errors.yaml +++ b/arkindex/project/tests/config_samples/errors.yaml @@ -27,8 +27,6 @@ features: gitlab: app_id: yes app_secret: [] -influxdb: - api_url: no internal_group_id: 2 ingest: endpoint: https://ohno diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml index 03575798ac16bb12928c0e35217509159cd55a1b..636a5e8b272cf9ee763aefc4b14f2b5e2363ea6e 100644 --- a/arkindex/project/tests/config_samples/override.yaml +++ b/arkindex/project/tests/config_samples/override.yaml @@ -55,8 +55,6 @@ gitlab: app_id: a app_secret: b imports_worker_version: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa -influxdb: - api_url: http://graph/ ingest: access_key_id: abcd endpoint: somewhere