support chunks in git export; rebase before merging to avoid merging manually

Closes #32 (closed)

arkindex_worker/git.py

@@ -6,6 +6,7 @@ from datetime import datetime
 from pathlib import Path
 
 import gitlab
+import requests
 import sh
 
 from arkindex_worker import logger
@@ -57,23 +58,20 @@ class GitlabHelper:
         :param title: title of the merge request
         :return: was the branch successfully merged?
         """
-        logger.info(f"Creating a merge request for {branch_name}")
-        mr = self.project.mergerequests.create(
-            {
-                "source_branch": branch_name,
-                "target_branch": self.branch,
-                "title": title,
-            }
-        )
+        mr = None
         # always rebase first, because other workers might have merged already
         for i in range(self.max_rebase_tries):
             logger.info(f"Trying to merge, try nr: {i}")
-            mr.rebase()
-            rebase_success = self._wait_for_rebase_to_finish(mr.iid)
-            if not rebase_success:
-                logger.error("Rebase failed, won't be able to merge!")
-                return False
             try:
+                if mr is None:
+                    mr = self._create_merge_request(branch_name, title)
+
+                mr.rebase()
+                rebase_success = self._wait_for_rebase_to_finish(mr.iid)
+                if not rebase_success:
+                    logger.error("Rebase failed, won't be able to merge!")
+                    return False
+
                 mr.merge(should_remove_source_branch=self.delete_source_branch)
                 logger.info("Merge successful")
                 return True
@@ -84,8 +82,30 @@ class GitlabHelper:
                 else:
                     logger.error(f"Merge was not successful: {e}")
                     return False
+            except gitlab.GitlabError as e:
+                logger.error(f"Gitlab error: {e}")
+                if 400 <= e.response_code < 500:
+                    # 4XX errors shouldn't be fixed by retrying
+                    raise e
+            except requests.exceptions.ConnectionError as e:
+                logger.error(f"Server connection error, will wait and retry: {e}")
+                time.sleep(self.rebase_wait_period)
+
         return False
 
+    def _create_merge_request(self, branch_name, title):
+        logger.info(f"Creating a merge request for {branch_name}")
+        # retry_transient_error will retry the request on 50X errors
+        # https://github.com/python-gitlab/python-gitlab/blob/265dbbdd37af88395574564aeb3fd0350288a18c/gitlab/__init__.py#L539
+        mr = self.project.mergerequests.create(
+            {
+                "source_branch": branch_name,
+                "target_branch": self.branch,
+                "title": title,
+            },
+        )
+        return mr
+
     def _get_merge_request(self, merge_request_id, include_rebase_in_progress=True):
         return self.project.mergerequests.get(
             merge_request_id, include_rebase_in_progress=include_rebase_in_progress
@@ -315,7 +335,11 @@ class GitHelper:
         self._git.push("-u", "origin", "HEAD")
 
         if self.gitlab_helper:
-            self.gitlab_helper.merge(branch_name, f"Merge {branch_name}")
+            try:
+                self.gitlab_helper.merge(branch_name, f"Merge {branch_name}")
+            except Exception as e:
+                logger.error(f"Merge failed: {e}")
+                raise e
         else:
             logger.info(
                 "No gitlab_helper defined, not trying to merge the pushed branch"