Merge branch 'offline-mode' into 'master'

Offline manifest importation

See merge request !21

arkindex/documents/importer.py

@@ -108,9 +108,8 @@ class ManifestsImporter(ABC):
     """
     Parses JSON manifests and annotation data to import them in the database.
     """
-    offline = False
 
-    def __init__(self, imgserv):
+    def __init__(self, imgserv, offline=False):
         """Initialize a manifest importer
         `imgserv` can be either one ImageServer or a list of ImageServers."""
         if isinstance(imgserv, ImageServer):
@@ -119,6 +118,8 @@ class ManifestsImporter(ABC):
             assert all(isinstance(srv, ImageServer) for srv in imgserv)
             self.imgserv = imgserv
 
+        self.offline = offline
+
         # This dictionary associates canvas IDs with images and pages
         # Filled by parse_manifest ; used by parse_annotation_list
         self.canvas_image = {}
@@ -138,9 +139,8 @@ class ManifestsImporter(ABC):
         """
         pass
 
-    def run(self, offline=False):
+    def run(self):
         """Start the importing process."""
-        self.offline = offline
         for f in self.get_json_files():
             self.parse_file(f)
             self.save_transcriptions()
@@ -223,7 +223,12 @@ class ManifestsImporter(ABC):
             # Create image and page
             try:
                 logger.debug("Looking for image {}".format(path))
-                image = image_server.find_image(path, offline=self.offline)
+                image = image_server.find_image(
+                    path,
+                    offline=self.offline,
+                    width=canvas['width'],
+                    height=canvas['height']
+                )
                 logger.debug("Found image {}".format(image.id))
                 logger.debug("Looking for page {}".format(folio))
                 page = import_page(volume, image, volume.name, folio, i)
@@ -352,8 +357,8 @@ class ManifestsImporter(ABC):
 class LocalManifestsImporter(ManifestsImporter):
     """Allows importing of local JSON files."""
 
-    def __init__(self, imgserv, path):
-        super().__init__(imgserv)
+    def __init__(self, imgserv, path, **kwargs):
+        super().__init__(imgserv, **kwargs)
         self.path = path
 
     def get_json_files(self):

arkindex/documents/management/commands/import_manifest.py

@@ -46,4 +46,8 @@ class Command(BaseCommand):
         except Exception as e:
             raise CommandError("Image server not found: {}".format(e))
 
-        LocalManifestsImporter(servers, options['manifest_folder']).run(offline=options['offline'])
+        LocalManifestsImporter(
+            servers,
+            options['manifest_folder'],
+            offline=options['offline']
+        ).run()

arkindex/images/admin.py

 from django.contrib import admin
 from arkindex.images.models import ImageServer, Image
+from enumfields.admin import EnumFieldListFilter
 
 
 class ImageServerAdmin(admin.ModelAdmin):
@@ -7,9 +8,12 @@ class ImageServerAdmin(admin.ModelAdmin):
 
 
 class ImageAdmin(admin.ModelAdmin):
-    list_display = ('path', 'server', 'width', 'height')
-    list_filter = ('server', )
-    fields = ('id', 'server', 'path', 'width', 'height')
+    list_display = ('path', 'server', 'width', 'height', 'status')
+    list_filter = (
+        ('server', admin.RelatedFieldListFilter),
+        ('status', EnumFieldListFilter),
+    )
+    fields = ('id', 'server', 'path', 'width', 'height', 'status')
     readonly_fields = ('id', )
 
 

arkindex/images/migrations/0002_image_status.py

+# Generated by Django 2.0 on 2018-05-15 10:33
+
+import arkindex.images.models
+from django.db import migrations
+import enumfields.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('images', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='image',
+            name='status',
+            field=enumfields.fields.EnumField(
+                default='checked',
+                enum=arkindex.images.models.ImageStatus,
+                max_length=50
+            ),
+        ),
+    ]

arkindex/images/models.py

@@ -7,6 +7,7 @@ from elasticsearch.helpers import bulk as es_bulk
 from arkindex.documents.models import Transcription
 from arkindex.project.models import IndexableModel
 from arkindex.project.tools import BoundingBox
+from enumfields import EnumField, Enum
 import requests
 import urllib.parse
 
@@ -24,7 +25,7 @@ class ImageServer(models.Model):
     def __str__(self):
         return self.name
 
-    def find_image(self, path, offline=False):
+    def find_image(self, path, offline=False, width=None, height=None):
         """
         Lookup an image on server
         This is the preferred way to construct an image
@@ -47,7 +48,8 @@ class ImageServer(models.Model):
 
         # Support offline queries
         if offline:
-            # TODO: create dumb image here
+            if img is None:
+                img = self.images.create(path=path, width=width, height=height)
             return img
 
         # Check the source
@@ -75,7 +77,9 @@ class ImageServer(models.Model):
             # Missing width or height data in image
             img.width = data['width']
             img.height = data['height']
-            img.save()
+
+        img.status = ImageStatus.Checked
+        img.save()
 
         return img
 
@@ -89,6 +93,15 @@ class ImageServer(models.Model):
         return urllib.parse.urljoin(base_url, urllib.parse.quote(path.lstrip('/')))
 
 
+class ImageStatus(Enum):
+    """
+    Image validation status
+    """
+    Checked = "checked"
+    Unchecked = "unchecked"
+    Error = "error"
+
+
 class Image(IndexableModel):
     """
     A document image
@@ -98,6 +111,7 @@ class Image(IndexableModel):
     width = models.PositiveIntegerField(default=0)
     height = models.PositiveIntegerField(default=0)
     elements = models.ManyToManyField('documents.Element', related_name='images', through='Zone')
+    status = EnumField(ImageStatus, default=ImageStatus.Unchecked, max_length=50)
 
     class Meta:
         unique_together = (