Merge branch 's3-uploads-with-mime-type' into 'master'

S3 uploads with mime type and check

See merge request !339

arkindex/dataimport/api.py

@@ -318,8 +318,15 @@ class DataFileUpload(CorpusACLMixin, APIView):
         )
 
         file_obj.open()
-        df.s3_object.upload_fileobj(file_obj.file)
-        df.save()
+        df.s3_object.upload_fileobj(
+            file_obj.file,
+            ExtraArgs={'ContentType': df.content_type}
+        )
+        try:
+            # If hash check is passed datafile instance will be saved
+            df.check_hash(save=True, raise_exc=True)
+        except ValueError:
+            raise ValidationError({'file': ['Remote server hash do not match given file hash']})
 
         return Response(
             data=DataFileSerializer(df).data,

arkindex/dataimport/tests/test_files.py

@@ -7,6 +7,7 @@ from arkindex.documents.models import Corpus
 from arkindex.dataimport.models import DataFile
 from arkindex.images.models import ImageServer
 from arkindex.project.tests import FixtureAPITestCase
+from arkindex.project.aws import S3FileStatus
 from arkindex.users.models import User
 import uuid
 
@@ -92,6 +93,7 @@ class TestFiles(FixtureAPITestCase):
         Assert a file upload creates a database instance and saves the file
         """
         f = SimpleUploadedFile('test.txt', b'This is a text file')
+        s3_mock.Object.return_value.e_tag = "bdc70a6c91dbb9b87a3bde0436bfd3f0"
 
         response = self.client.post(reverse('api:file-upload', kwargs={'pk': self.corpus.id}), data={'file': f})
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
@@ -103,6 +105,8 @@ class TestFiles(FixtureAPITestCase):
         self.assertEqual(df.size, 19)
         self.assertEqual(df.content_type, 'text/plain')
         self.assertEqual(df.corpus, self.corpus)
+        self.assertEqual(df.hash, s3_mock.Object.return_value.e_tag)
+        self.assertEqual(df.status, S3FileStatus.Checked)
         self.assertEqual(s3_mock.Object.call_count, 1)
         self.assertEqual(s3_mock.Object.call_args, call('staging', str(df.id)))
         self.assertEqual(s3_mock.Object().upload_fileobj.call_count, 1)
@@ -130,6 +134,7 @@ class TestFiles(FixtureAPITestCase):
         Assert file upload does not trust the client defined content type
         """
         f = SimpleUploadedFile('test.mp3', b'This actually is a text file', content_type='audio/mpeg')
+        s3_mock.Object.return_value.e_tag = "a92f087e06876600af579a472ab4db96"
 
         response = self.client.post(reverse('api:file-upload', kwargs={'pk': self.corpus.id}), data={'file': f})
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
@@ -139,6 +144,7 @@ class TestFiles(FixtureAPITestCase):
         df = DataFile.objects.get(id=data['id'])
         self.assertEqual(df.name, 'test.mp3')
         self.assertEqual(df.content_type, 'text/plain')
+        self.assertEqual(df.status, S3FileStatus.Checked)
 
         self.assertEqual(s3_mock.Object.call_count, 1)
         self.assertEqual(s3_mock.Object().upload_fileobj.call_count, 1)
@@ -149,6 +155,7 @@ class TestFiles(FixtureAPITestCase):
         Assert uploading the same file twice fails
         """
         f = SimpleUploadedFile('test.txt', b'This is a text file')
+        s3_mock.Object.return_value.e_tag = "bdc70a6c91dbb9b87a3bde0436bfd3f0"
         response = self.client.post(reverse('api:file-upload', kwargs={'pk': self.corpus.id}), data={'file': f})
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         data = response.json()

arkindex/images/tests/test_image.py

@@ -71,9 +71,18 @@ class TestImage(FixtureTestCase):
         s3_mock.Object().e_tag = '"beef"'
         img = ImageServer.objects.local.images.create(path='abcd', hash='beef')
         self.assertEqual(img.status, S3FileStatus.Unchecked)
-        img.check_hash()
+        img.check_hash(save=False)
+        img.refresh_from_db()
         self.assertEqual(img.status, S3FileStatus.Unchecked)
 
+    @patch('arkindex.project.aws.s3')
+    def test_check_hash_save(self, s3_mock):
+        s3_mock.Object().e_tag = '"beef"'
+        img = ImageServer.objects.local.images.create(path='abcd', hash='beef')
+        img.check_hash(save=True)
+        img.refresh_from_db()
+        self.assertEqual(img.status, S3FileStatus.Checked)
+
     @override_settings(LOCAL_IMAGESERVER_ID=-1)
     @patch('arkindex.project.aws.s3')
     def test_check_hash_external(self, s3_mock):

arkindex/project/aws.py

@@ -89,11 +89,12 @@ class S3FileMixin(object):
         assert self.exists(), 'No file content, assert file has been correctly uploaded'
         # The hash given by Boto seems to be surrounded by double quotes
         if self.s3_object.e_tag.strip('"') == self.hash:
-            return
-        self.status = S3FileStatus.Error
+            self.status = S3FileStatus.Checked
+        else:
+            self.status = S3FileStatus.Error
         if save:
             self.save()
-        if raise_exc:
+        if self.status == S3FileStatus.Error and raise_exc:
             raise ValueError('MD5 hashes do not match')