From 080bce2f86b24f9f8a6d558ac6048fbe42fc5ee5 Mon Sep 17 00:00:00 2001
From: Yoann Schneider <yschneider@teklia.com>
Date: Wed, 30 Nov 2022 17:18:24 +0100
Subject: [PATCH] Revert "Only use files when computing model archive hash"
This reverts commit ceb2adbcca88fecfa8f75d31a320ce7594212e62.
---
arkindex_worker/worker/training.py | 8 +++++---
tests/conftest.py | 7 +------
.../model-best/model_file.pth | 1 -
.../model-last/model_file.pth | 1 -
tests/test_elements_worker/test_training.py | 18 ------------------
5 files changed, 6 insertions(+), 29 deletions(-)
delete mode 100644 tests/samples/model_files_with_subfolder/model-best/model_file.pth
delete mode 100644 tests/samples/model_files_with_subfolder/model-last/model_file.pth
diff --git a/arkindex_worker/worker/training.py b/arkindex_worker/worker/training.py
index ffe52d6f..c53cffee 100644
--- a/arkindex_worker/worker/training.py
+++ b/arkindex_worker/worker/training.py
@@ -50,10 +50,12 @@ def create_archive(path: DirPath) -> Tuple[Path, Hash, FileSize, Hash]:
# Create an uncompressed tar archive with all the needed files
# Files hierarchy ifs kept in the archive.
-
+ file_list = []
with tarfile.open(path_to_tar_archive, "w") as tar:
- tar.add(path)
- file_list = [member for member in tar.getnames() if os.path.isfile(member)]
+ for p in path.glob("**/*"):
+ x = p.relative_to(path)
+ tar.add(p, arcname=x, recursive=False)
+ file_list.append(p)
# Sort by path
file_list.sort()
diff --git a/tests/conftest.py b/tests/conftest.py
index 8b6f16ef..d058dc87 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -26,7 +26,7 @@ from arkindex_worker.worker import BaseWorker, ElementsWorker
from arkindex_worker.worker.transcription import TextOrientation
FIXTURES_DIR = Path(__file__).resolve().parent / "data"
-SAMPLES_DIR = Path("tests") / "samples"
+SAMPLES_DIR = Path(__file__).resolve().parent / "samples"
__yaml_cache = {}
@@ -280,11 +280,6 @@ def model_file_dir():
return SAMPLES_DIR / "model_files"
-@pytest.fixture
-def model_file_dir_with_subfolder():
- return SAMPLES_DIR / "model_files_with_subfolder"
-
-
@pytest.fixture
def fake_dummy_worker():
api_client = MockApiClient()
diff --git a/tests/samples/model_files_with_subfolder/model-best/model_file.pth b/tests/samples/model_files_with_subfolder/model-best/model_file.pth
deleted file mode 100644
index cc78ba30..00000000
--- a/tests/samples/model_files_with_subfolder/model-best/model_file.pth
+++ /dev/null
@@ -1 +0,0 @@
-Wow this is actually the data of the best model ever created on Arkindex
\ No newline at end of file
diff --git a/tests/samples/model_files_with_subfolder/model-last/model_file.pth b/tests/samples/model_files_with_subfolder/model-last/model_file.pth
deleted file mode 100644
index cc78ba30..00000000
--- a/tests/samples/model_files_with_subfolder/model-last/model_file.pth
+++ /dev/null
@@ -1 +0,0 @@
-Wow this is actually the data of the best model ever created on Arkindex
\ No newline at end of file
diff --git a/tests/test_elements_worker/test_training.py b/tests/test_elements_worker/test_training.py
index 4448554d..e8f41962 100644
--- a/tests/test_elements_worker/test_training.py
+++ b/tests/test_elements_worker/test_training.py
@@ -45,24 +45,6 @@ def test_create_archive(model_file_dir):
assert not os.path.exists(zst_archive_path), "Auto removal failed"
-def test_create_archive_with_subfolder(model_file_dir_with_subfolder):
- """Create an archive when the model's file is in a folder containing a subfolder"""
-
- with create_archive(path=model_file_dir_with_subfolder) as (
- zst_archive_path,
- hash,
- size,
- archive_hash,
- ):
- assert os.path.exists(zst_archive_path), "The archive was not created"
- assert (
- hash == "e2fa86cefc33b24502ad4151a638dd29"
- ), "Hash was not properly computed"
- assert 300 < size < 1500
-
- assert not os.path.exists(zst_archive_path), "Auto removal failed"
-
-
@pytest.mark.parametrize(
"tag, description",
[
--
GitLab