publishing a folder containing subfolders

arkindex_worker/worker/training.py

@@ -30,7 +30,9 @@ FileSize = NewType("FileSize", int)
 
 
 @contextmanager
-def create_archive(path: DirPath) -> Tuple[Path, Hash, FileSize, Hash]:
+def create_archive(
+    path: DirPath, use_parent_folder: bool = False
+) -> Tuple[Path, Hash, FileSize, Hash]:
     """
     Create a tar archive from the files at the given location then compress it to a zst archive.
 
@@ -52,10 +54,12 @@ def create_archive(path: DirPath) -> Tuple[Path, Hash, FileSize, Hash]:
     # Files hierarchy ifs kept in the archive.
     file_list = []
     with tarfile.open(path_to_tar_archive, "w") as tar:
-        for p in path.glob("**/*"):
-            x = p.relative_to(path)
-            tar.add(p, arcname=x, recursive=False)
-            file_list.append(p)
+        if use_parent_folder:
+            # Only publish contents of the given folder without parent folder
+            tar.add(path, arcname=".")
+        else:
+            tar.add(path)
+        file_list = [member for member in tar.getnames() if os.path.isfile(member)]
 
     # Sort by path
     file_list.sort()