Remove OAuthCredentials, Repository.credentials, Gitlab providers and dependency

README.md

@@ -43,26 +43,6 @@ You will need to edit the ImageMagick policy file to get PDF and Image imports t
 
 The line that sets the PDF policy is `<policy domain="coder" rights="none" pattern="PDF" />`. Replace `none` with `read|write` for it to work. See [this StackOverflow question](https://stackoverflow.com/questions/52998331) for more info.
 
-### GitLab OAuth setup
-
-Arkindex uses OAuth to let a user connect their GitLab account(s) and register Git repositories. In local development, you will need to register Arkindex as a GitLab OAuth application for it to work.
-
-Go to GitLab's [Applications settings](https://gitlab.teklia.com/profile/applications) and create a new application with the `api` scope and add the following callback URIs:
-
-```
-http://127.0.0.1:8000/api/v1/oauth/providers/gitlab/callback/
-http://ark.localhost:8000/api/v1/oauth/providers/gitlab/callback/
-https://ark.localhost/api/v1/oauth/providers/gitlab/callback/
-```
-
-Once the application is created, GitLab will provide you with an application ID and a secret. Use the `arkindex/config.yml` file to set them:
-
-```yaml
-gitlab:
-  app_id: 24cacf5004bf68ae9daad19a5bba391d85ad1cb0b31366e89aec86fad0ab16cb
-  app_secret: 9d96d9d5b1addd7e7e6119a23b1e5b5f68545312bfecb21d1cdc6af22b8628b8
-```
-
 ### Local image server
 
 Arkindex splits up image URLs in their image server and the image path. For example, a IIIF server at `http://iiif.irht.cnrs.fr/iiif/` and an image at `/Paris/JJ042/1.jpg` would be represented as an ImageServer instance holding one Image. Since Arkindex has a local IIIF server for image uploads and thumbnails, a special instance of ImageServer is required to point to this local server. In local development, this server should be available at `https://ark.localhost/iiif`. You will therefore need to create an ImageServer via the Django admin or the Django shell with this URL. To set the local server ID, you can add a custom setting in `arkindex/config.yml`:
@@ -161,9 +141,6 @@ SHELL_PLUS_POST_IMPORTS = [
     )),
     ('arkindex.project.aws', (
         'S3FileStatus',
-    )),
-    ('arkindex.users.models', (
-        'OAuthStatus',
     ))
 ]
 ```

arkindex/documents/management/commands/bootstrap.py

@@ -158,9 +158,6 @@ class Command(BaseCommand):
         # Create a fake worker version on a fake worker on a fake repo with a fake revision for file imports
         repo, created = Repository.objects.get_or_create(
             url=IMPORT_WORKER_REPO,
-            defaults={
-                "hook_token": str(uuid4()),
-            }
         )
         if created:
             self.success(f'Created Git repository for {IMPORT_WORKER_REPO}')

arkindex/documents/management/commands/build_fixtures.py

@@ -13,6 +13,7 @@ from arkindex.process.models import (
     FeatureUsage,
     Process,
     ProcessMode,
+    Repository,
     Worker,
     WorkerRun,
     WorkerType,
@@ -69,19 +70,9 @@ class Command(BaseCommand):
             level=Role.Guest.value
         )
 
-        # Create OAuth credentials for a user
-        creds = user.credentials.create(
-            provider_url='https://somewhere',
-            token='oauth-token',
-            refresh_token='refresh-token',
-            # Use an expiry very far away to avoid OAuth token refreshes in every test
-            expiry=datetime(2100, 12, 31, 23, 59, 59, 999999, timezone.utc),
-        )
-
         # Create a GitLab worker repository
-        gitlab_repo = creds.repos.create(
+        gitlab_repo = Repository.objects.create(
             url='http://gitlab/repo',
-            hook_token='hook-token',
         )
 
         # Create a revision on this repository
@@ -92,9 +83,8 @@ class Command(BaseCommand):
         )
 
         # Create another worker repository
-        worker_repo = creds.repos.create(
+        worker_repo = Repository.objects.create(
             url="http://my_repo.fake/workers/worker",
-            hook_token='worker-hook-token',
         )
 
         # Create a revision on this repository

arkindex/documents/management/commands/load_export.py

@@ -2,7 +2,6 @@
 import json
 import os
 import sqlite3
-import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -410,9 +409,6 @@ class Command(BaseCommand):
     def create_repository(self, row):
         repo, created = Repository.objects.get_or_create(
             url=row['repository_url'],
-            defaults={
-                'hook_token': str(uuid.uuid4()),
-            },
         )
         return repo, created
 

arkindex/documents/tests/commands/test_load_export.py

@@ -33,7 +33,7 @@ class TestLoadExport(FixtureTestCase):
         unexpected_fields_by_model = {
             'documents.elementtype': ['display_name', 'indexable'],
             'documents.mlclass': [],
-            'process.repository': ['hook_token', 'credentials', 'git_ref_revisions'],
+            'process.repository': ['git_ref_revisions'],
             'process.worker': [],
             'process.revision': ['message', 'author'],
             'process.workerversion': ['created', 'updated', 'configuration', 'state', 'docker_image', 'docker_image_iid'],

arkindex/documents/tests/tasks/test_corpus_delete.py

@@ -96,7 +96,6 @@ class TestDeleteCorpus(FixtureTestCase):
         # Create a separate corpus that should not get anything deleted
         cls.repo = Repository.objects.create(
             url='http://lol.git',
-            hook_token='h00k',
         )
         cls.corpus2 = Corpus.objects.create(name='Other corpus')
 

arkindex/process/admin.py

@@ -76,7 +76,7 @@ class WorkerInline(admin.StackedInline):
 
 class RepositoryAdmin(admin.ModelAdmin):
     list_display = ('id', 'url')
-    fields = ('id', 'url', 'hook_token', 'credentials')
+    fields = ('id', 'url')
     readonly_fields = ('id', )
     inlines = [WorkerInline, UserMembershipInline, GroupMembershipInline]
 

arkindex/process/api.py

@@ -5,7 +5,6 @@ from textwrap import dedent
 from uuid import UUID
 
 from django.conf import settings
-from django.core.mail import send_mail
 from django.db import transaction
 from django.db.models import (
     Avg,
@@ -24,7 +23,6 @@ from django.db.models import (
 from django.db.models.functions import Coalesce, Now
 from django.db.models.query import Prefetch
 from django.shortcuts import get_object_or_404
-from django.template.loader import render_to_string
 from django.utils.functional import cached_property
 from drf_spectacular.utils import (
     OpenApiExample,
@@ -51,7 +49,6 @@ from rest_framework.generics import (
 )
 from rest_framework.response import Response
 from rest_framework.serializers import Serializer
-from rest_framework.views import APIView
 
 from arkindex.documents.models import Corpus, Element, Selection
 from arkindex.ponos.authentication import TaskAuthentication
@@ -65,7 +62,6 @@ from arkindex.process.models import (
     Process,
     ProcessDataset,
     ProcessMode,
-    Repository,
     Revision,
     Worker,
     WorkerActivity,
@@ -75,9 +71,8 @@ from arkindex.process.models import (
     WorkerType,
     WorkerVersion,
 )
-from arkindex.process.providers import GitProvider
 from arkindex.process.serializers.files import DataFileCreateSerializer, DataFileSerializer
-from arkindex.process.serializers.git import ExternalRepositorySerializer, RevisionSerializer
+from arkindex.process.serializers.git import RevisionSerializer
 from arkindex.process.serializers.imports import (
     ApplyProcessTemplateSerializer,
     CorpusProcessSerializer,
@@ -131,7 +126,7 @@ from arkindex.project.tools import PercentileCont
 from arkindex.project.triggers import process_delete
 from arkindex.training.models import Dataset, Model
 from arkindex.training.serializers import DatasetSerializer
-from arkindex.users.models import OAuthCredentials, Role, Scope
+from arkindex.users.models import Role, Scope
 
 logger = logging.getLogger(__name__)
 
@@ -448,8 +443,6 @@ class ProcessRetry(ProcessACLMixin, ProcessQuerysetMixin, GenericAPIView):
         if process.mode == ProcessMode.Repository:
             if not process.revision:
                 raise ValidationError({'__all__': ['Git repository imports must have a revision set']})
-            if not process.revision.repo.enabled:
-                raise ValidationError({'__all__': ['Git repository does not have any valid credentials']})
 
     @extend_schema(
         operation_id='RetryProcess',
@@ -773,45 +766,6 @@ class ProcessDatasetManage(CreateAPIView, DestroyAPIView):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-@extend_schema(exclude=True)
-class GitRepositoryImportHook(APIView):
-    """
-    This endpoint is intended as a webhook for Git repository hosting applications like GitLab.
-    """
-
-    def post(self, request, pk=None, **kwargs):
-        repo = get_object_or_404(Repository, id=pk)
-        if not repo.enabled:
-            raise PermissionDenied(detail='No credentials available for this repository.')
-
-        try:
-            repo.provider_class(credentials=repo.credentials).handle_webhook(repo, request)
-        except Exception as e:
-            user = repo.credentials.user
-            # Notifying the user by mail that there was an error during webhook handling
-            sent = send_mail(
-                subject='An error occurred during Git hook handling',
-                message=render_to_string(
-                    'webhook_error.html',
-                    context={
-                        'user': user,
-                        'url': repo.url,
-                        'error': e.args[0],
-                    },
-                    request=request,
-                ),
-                from_email=None,
-                recipient_list=[user.email],
-                fail_silently=True,
-            )
-            if sent == 0:
-                logger.error(f'Failed to send webhook error email to {user.email}')
-
-            raise
-
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
 @extend_schema_view(
     get=extend_schema(
         tags=['repos'],
@@ -829,75 +783,10 @@ class RepositoryList(RepositoryACLMixin, ListAPIView):
     def get_queryset(self):
         return self.readable_repositories \
             .annotate(authorized_users=Count('memberships')) \
-            .select_related('credentials') \
             .prefetch_related('workers__type') \
             .order_by('url')
 
 
-@extend_schema_view(
-    get=extend_schema(
-        operation_id='ListExternalRepositories',
-        parameters=[
-            OpenApiParameter(
-                'search',
-                description='Optional query terms to filter repositories',
-            )
-        ],
-        tags=['repos'],
-    ),
-    post=extend_schema(
-        operation_id='CreateExternalRepository',
-        description='Using the given OAuth credentials, this links an external Git repository '
-                    'to Arkindex, connects a push hook and starts an initial process.',
-        responses={201: ProcessSerializer},
-        tags=['repos'],
-    )
-)
-class AvailableRepositoriesList(ListCreateAPIView):
-    """
-    List repositories associated to user OAuth credentials
-
-    Using the given OAuth credentials ID, this uses the Git hosting
-    application API's search feature to look for a repository matching
-    the given query. Without a query, returns a full list.
-    """
-    permission_classes = (IsVerified, )
-    pagination_class = None
-    serializer_class = ExternalRepositorySerializer
-    queryset = OAuthCredentials.objects.none()
-
-    def get_queryset(self):
-        cred = get_object_or_404(OAuthCredentials, user=self.request.user, id=self.kwargs['pk'])
-        provider = cred.git_provider_class(credentials=cred)
-        repos = provider.list_repos(query=self.request.query_params.get('search'))
-        return map(provider.to_dict, repos)
-
-    @transaction.atomic
-    def create(self, request, *args, **kwargs):
-        serializer = self.get_serializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        user = self.request.user
-        cred = get_object_or_404(OAuthCredentials, user=user, id=self.kwargs['pk'])
-        provider = cred.git_provider_class(credentials=cred)
-        repo = provider.create_repo(
-            request=self.request,
-            id=serializer.validated_data.get('id')
-        )
-        # Add an admin membership to the repository creator
-        repo.memberships.create(user=user, level=Role.Admin.value)
-
-        latest_commit_sha = provider.get_latest_commit_sha(repo)
-        rev, _ = provider.get_or_create_revision(repo, latest_commit_sha)
-
-        process = GitProvider.start_imports(provider, rev)
-
-        # Return the serialized process
-        return Response(
-            status=status.HTTP_201_CREATED,
-            data=ProcessSerializer(process, context={'request': self.request}).data,
-        )
-
-
 @extend_schema(
     tags=['repos'],
     description='Retrieve a repository',

arkindex/process/builder.py

@@ -187,8 +187,6 @@ class ProcessBuilder(object):
     def validate_repository(self) -> None:
         if self.process.revision is None:
             raise ValidationError('A revision is required to create an import workflow from GitLab repository')
-        if not self.process.revision.repo.enabled:
-            raise ValidationError('Git repository does not have any valid credentials')
 
     def validate_s3(self) -> None:
         if not self.process.bucket_name:

arkindex/process/management/commands/update_repositories_hooks.py

-from django.core.management.base import BaseCommand
-
-from arkindex.process.models import Repository
-
-
-class Command(BaseCommand):
-    help = "Update all Git repositories hooks"
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            "base_url",
-            type=str,
-            help="Base url used to build the new hooks",
-        )
-
-    def handle(self, base_url, *args, **kwargs):
-        for repository in Repository.objects.filter(credentials__isnull=False):
-            self.stdout.write(f"Updating {repository}")
-            repository.provider.create_hook(repository, base_url=base_url)
-
-        self.stdout.write(self.style.SUCCESS("All done"))

arkindex/process/management/commands/update_repositories_refs.py

-from django.core.management.base import BaseCommand
-
-from arkindex.process.models import Repository
-
-
-class Command(BaseCommand):
-    help = "Update all Git repositories references and trigger build processes"
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            "--repo-url",
-            type=str,
-            help="Limit process to one repository",
-        )
-
-    def handle(self, repo_url, *args, **kwargs):
-
-        repositories = Repository.objects.all()
-        if repo_url:
-            repositories = repositories.filter(url=repo_url)
-
-        for repository in repositories:
-
-            self.stdout.write(f"Updating refs for {repository}")
-
-            # List all refs
-            try:
-                repository.provider.update_repository_references(repository)
-            except Exception as e:
-                self.stderr.write(f"Failure for {repository}: {e}")
-
-        self.stdout.write("All done")

arkindex/process/migrations/0026_remove_repository_unique_repository_hook_token_and_more.py

+# Generated by Django 4.1.7 on 2023-12-20 11:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('process', '0025_worker_archived'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            "ALTER TABLE process_repository DROP CONSTRAINT unique_repository_hook_token",
+            elidable=True,
+        ),
+        migrations.RemoveConstraint(
+            model_name='repository',
+            name='unique_repository_hook_token',
+        ),
+        migrations.RemoveField(
+            model_name='repository',
+            name='credentials',
+        ),
+        migrations.RemoveField(
+            model_name='repository',
+            name='hook_token',
+        ),
+    ]

arkindex/process/models.py

@@ -23,7 +23,6 @@ from arkindex.process.managers import (
     WorkerRunManager,
     WorkerVersionManager,
 )
-from arkindex.process.providers import get_provider
 from arkindex.project.aws import S3FileMixin, S3FileStatus
 from arkindex.project.fields import ArrayField, MD5HashField
 from arkindex.project.models import IndexableModel
@@ -549,9 +548,6 @@ class DataFile(S3FileMixin, models.Model):
 class Repository(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     url = models.URLField()
-    hook_token = models.CharField(max_length=250)
-    credentials = models.ForeignKey(
-        'users.OAuthCredentials', on_delete=models.SET_NULL, related_name='repos', blank=True, null=True)
     git_ref_revisions = models.ManyToManyField('process.Revision', through='process.GitRef')
     memberships = GenericRelation('users.Right', 'content_id')
 
@@ -561,32 +557,9 @@ class Repository(models.Model):
             models.UniqueConstraint(
                 'url',
                 name='unique_repository_url',
-            ),
-            models.UniqueConstraint(
-                'hook_token',
-                name='unique_repository_hook_token',
-            ),
+            )
         ]
 
-    # Make this constant into a database field to be able to handle repositories from services other than GitLab
-    provider_name = 'GitLabProvider'
-
-    @property
-    def provider_class(self):
-        if not self.provider_name:
-            raise ValueError("Empty provider_name")
-        return get_provider(self.provider_name)
-
-    @property
-    def provider(self):
-        if self.provider_class is None:
-            raise NotImplementedError(f"Missing provider {self.provider_name}")
-        return self.provider_class(credentials=self.credentials)
-
-    @property
-    def enabled(self):
-        return self.credentials is not None and self.provider_class is not None
-
     def __str__(self):
         return f'{self.url}'
 

arkindex/process/providers.py

-import base64
-import logging
-import urllib.parse
-import uuid
-from abc import ABC, abstractmethod
-
-from django.conf import settings
-from django.urls import reverse
-from gitlab import Gitlab, GitlabCreateError, GitlabGetError
-from rest_framework.exceptions import APIException, AuthenticationFailed, NotAuthenticated, ValidationError
-
-from arkindex.process.utils import get_default_farm
-
-logger = logging.getLogger(__name__)
-
-
-class GitProvider(ABC):
-
-    display_name = None
-
-    def __init__(self, credentials=None):
-        if credentials:
-            from arkindex.users.models import OAuthCredentials
-            assert isinstance(credentials, OAuthCredentials)
-        self.credentials = credentials
-
-    @abstractmethod
-    def list_repos(self, query=None):
-        """
-        List all repositories or filter with a search query.
-        """
-
-    @abstractmethod
-    def get_latest_commit_sha(self, repo):
-        """
-        Retrieve latest commit sha from the default branch on a given repository.
-        """
-
-    @abstractmethod
-    def create_repo(self, **kwargs):
-        """
-        Create a Repository instance from an external repository
-        """
-
-    @abstractmethod
-    def create_hook(self, repository, project_id=None, base_url=None):
-        """Create a webhook to receive events from the project"""
-
-    def update_or_create_ref(self, repo, revision, name, type):
-        """
-        Update or create a GitRef on a given repository:
-        - If a GitRef with given name already exists on the given repository, we only update the
-        linked revision with the given one
-        - Else a GitRef is created on the given repository thanks to given name, type and revision
-        """
-        from arkindex.process.models import GitRef
-        try:
-            ref = repo.refs.get(name=name)
-            ref.revision = revision
-            ref.save()
-        except GitRef.DoesNotExist:
-            # Limit the name length to the max size of GitRef name field
-            repo.refs.create(name=name[:250], type=type, revision=revision)
-
-    def get_or_create_revision(self, repo, sha, save=True):
-        from arkindex.process.models import Revision
-        try:
-            return self.get_revision(repo, sha), False
-        except Revision.DoesNotExist:
-            return self.create_revision(repo, sha, save=save), True
-
-    def get_revision(self, repo, sha):
-        return repo.revisions.get(hash=sha)
-
-    @abstractmethod
-    def create_revision(self, repo, sha, save=True):
-        """
-        Create a Revision instance for a given commit hash of a given repository.
-        """
-
-    @abstractmethod
-    def handle_webhook(self, repo, request):
-        """
-        Handle a webhook event on a given repository.
-        """
-
-    @abstractmethod
-    def to_dict(self, project):
-        """
-        Returns a dict specifying id, name and url from a repository
-        """
-
-    def start_imports(self, rev):
-        """
-        Create and start a process to build new worker(s) from a repository revision
-        """
-        from arkindex.process.models import ProcessMode
-        mode = ProcessMode.Repository
-        user = rev.repo.credentials.user
-        refs = ", ".join(rev.refs.values_list('name', flat=True))
-        if not refs:
-            refs = rev.hash
-        # Limit the name length to the max size of Process name field
-        name = f"Import {refs} from {rev.repo.url}"[:100]
-
-        farm = get_default_farm()
-        if not farm.is_available(user):
-            raise ValidationError("The owner of the OAuth credentials does not have access to the default farm.")
-
-        process = rev.processes.create(
-            creator=user,
-            mode=mode,
-            name=name,
-            farm=farm,
-        )
-        process.run()
-        return process
-
-
-class GitLabProvider(GitProvider):
-
-    display_name = "GitLab"
-
-    def _get_gitlab_client(self, credentials):
-        if credentials.expired:
-            credentials.provider.refresh_token()
-
-        return Gitlab(credentials.provider_url, oauth_token=credentials.token)
-
-    def _try_get_project(self, gl, id):
-        try:
-            return gl.projects.get(id)
-        except GitlabGetError as e:
-            raise APIException("Error while fetching GitLab project: {}".format(str(e)))
-
-    def _get_project_from_repo(self, repo):
-        assert repo.credentials, "Missing Gitlab credentials"
-
-        gl = self._get_gitlab_client(repo.credentials)
-        return self._try_get_project(gl, urllib.parse.urlsplit(repo.url).path.strip('/'))
-
-    def list_repos(self, query=None):
-        if not self.credentials:
-            raise NotAuthenticated
-        gl = self._get_gitlab_client(self.credentials)
-        # Creating a webhook on a repo requires Maintainer (40) or Owner (50) access levels
-        # See https://docs.gitlab.com/ce/api/members.html#valid-access-levels
-        return gl.projects.list(min_access_level=40, search=query, get_all=False)
-
-    def get_latest_commit_sha(self, repo):
-        project = self._get_project_from_repo(repo)
-        # Since ref_name isn't specified, we will use the repository default branch
-        # See : https://docs.gitlab.com/ee/api/commits.html#list-repository-commits
-        return project.commits.list(per_page=1, get_all=False)[0].id
-
-    def to_dict(self, project):
-        return {
-            'id': project.id,
-            'name': project.name_with_namespace,
-            'url': project.web_url
-        }
-
-    def create_repo(self, id=None, **kwargs):
-        if not self.credentials:
-            raise NotAuthenticated()
-
-        gl = self._get_gitlab_client(self.credentials)
-        project = self._try_get_project(gl, int(id))
-
-        from arkindex.process.models import Repository
-        if Repository.objects.filter(url=project.web_url).exists():
-            raise ValidationError("A repository with this URL already exists")
-
-        # Determine the user's access level on this project
-        # When it is inherited from the group and not overridden in the project, project_access is not defined.
-        # When the project isn't in a group, or the user is added to a specific project in a group,
-        # group_access is not defined. project_access overrides group_access.
-        access_level = 0
-        if project.permissions.get('group_access'):
-            access_level = project.permissions['group_access']['access_level']
-        if project.permissions.get('project_access'):
-            access_level = project.permissions['project_access']['access_level']
-
-        # Maintainer level (40) is required
-        if access_level < 40:
-            raise ValidationError("Maintainer or Owner access is required to add a GitLab repository")
-
-        repo = self.credentials.repos.create(
-            url=project.web_url,
-            hook_token=base64.b64encode(uuid.uuid4().bytes).decode('utf-8'),
-        )
-
-        self.create_hook(repo, project_id=int(id))
-
-        return repo
-
-    def create_hook(self, repository, project_id=None, base_url=None):
-        """
-        Configure the Gitlab hook to get events for a project.
-
-        If `project_id` is set, then the project is retrieved directly from its GitLab project ID
-        instead of matched using its path.
-
-        The webhook's URL will use `base_url` as its base URL if it is set, and otherwise
-        falls back on settings.BACKEND_PUBLIC_URL_OAUTH first, then settings.PUBLIC_HOSTNAME.
-        """
-        # Load project using a project ID or its repo path
-        gitlab = Gitlab(repository.credentials.provider_url, oauth_token=repository.credentials.token)
-        if project_id:
-            project = self._try_get_project(gitlab, project_id)
-        else:
-            path = urllib.parse.urlparse(repository.url).path[1:]
-            project = self._try_get_project(gitlab, path)
-
-        if base_url is None:
-            base_url = settings.BACKEND_PUBLIC_URL_OAUTH or settings.PUBLIC_HOSTNAME
-
-        if base_url is None:
-            raise APIException('Either the `base_url` argument, settings.BACKEND_PUBLIC_URL_OAUTH or settings.PUBLIC_HOSTNAME must be set.')
-
-        url = urllib.parse.urljoin(base_url, reverse('api:import-hook', kwargs={'pk': repository.id}))
-        logger.info(f"Webhook will be created as {url}")
-
-        # Delete already configured hooks to be able to update
-        for hook in project.hooks.list(all=True):
-            if hook.url == url:
-                hook.delete()
-                logger.info(f"Deleted existing hook {hook.id}")
-
-        try:
-            # Create a new hook
-            hook = project.hooks.create({
-                'url': url,
-                'push_events': True,
-                "tag_push_events": True,
-                'token': repository.hook_token,
-            })
-            logger.info(f"Created new hook {hook.id}")
-        except GitlabCreateError as e:
-            raise APIException("Error while creating GitLab hook: {}".format(str(e)))
-
-    def create_revision(self, repo, sha, save=True):
-        from arkindex.process.models import GitRefType, Revision
-
-        project = self._get_project_from_repo(repo)
-        commit = project.commits.get(sha)
-
-        rev = Revision(
-            repo=repo,
-            hash=sha,
-            message=commit.message,
-            author=commit.author_name,
-        )
-        if save:
-            rev.save()
-            for ref in commit.refs():
-                try:
-                    ref_type = GitRefType(ref['type'])
-                    self.update_or_create_ref(repo, rev, ref['name'], ref_type)
-                except ValueError:
-                    logger.warning(f'Git reference with type {ref["type"]} was ignored during revision creation')
-                    continue
-
-        return rev
-
-    def update_revision_references(self, repo, sha):
-        """
-        Update all references for a specific revision
-        This is needed when a tag is pushed after its commit
-        is already ingested
-        """
-        from arkindex.process.models import GitRefType
-
-        project = self._get_project_from_repo(repo)
-        commit = project.commits.get(sha)
-
-        rev = repo.revisions.get(hash=sha)
-        for ref in commit.refs():
-            try:
-                ref_type = GitRefType(ref['type'])
-                self.update_or_create_ref(repo, rev, ref['name'], ref_type)
-            except ValueError:
-                logger.warning(f'Git reference with type {ref["type"]} was ignored during revision creation')
-                continue
-
-    def update_repository_references(self, repo):
-        """
-        List all available references (branches and tags) on a remote repository
-        and create new references or update existing ones
-        New import process are started for new references
-        """
-        from arkindex.process.models import GitRefType
-        project = self._get_project_from_repo(repo)
-
-        def _update_reference(ref, ref_type):
-            assert isinstance(ref_type, GitRefType)
-            try:
-                rev, created = repo.revisions.get_or_create(
-                    hash=ref.commit['id'],
-                    defaults={
-                        'message': ref.commit['message'],
-                        'author': ref.commit['author_email'],
-                    }
-                )
-                self.update_or_create_ref(repo, rev, ref.name, ref_type)
-                if created:
-                    logger.info(f"Starting import for {ref_type.value} {ref.name}")
-                    self.start_imports(rev)
-            except ValueError:
-                logger.warning(f'{ref_type.value} {ref.name} was ignored during revision creation')
-
-        for branch in project.branches.list(get_all=True):
-            _update_reference(branch, GitRefType.Branch)
-
-        for tag in project.tags.list(get_all=True):
-            _update_reference(tag, GitRefType.Tag)
-
-    def handle_webhook(self, repo, request):
-        from arkindex.process.models import GitRefType
-
-        if 'HTTP_X_GITLAB_EVENT' not in request.META:
-            raise ValidationError("Missing GitLab event type")
-
-        if request.META['HTTP_X_GITLAB_EVENT'] not in ('Push Hook', 'Tag Push Hook'):
-            raise ValidationError("Unsupported GitLab event type")
-
-        if 'HTTP_X_GITLAB_TOKEN' not in request.META:
-            raise NotAuthenticated("Missing GitLab secret token")
-        if request.META['HTTP_X_GITLAB_TOKEN'] != repo.hook_token:
-            raise AuthenticationFailed("Invalid GitLab secret token")
-
-        if not isinstance(request.data, dict) or 'object_kind' not in request.data:
-            raise ValidationError('Bad payload format')
-
-        kind = request.data.get('object_kind')
-        if kind not in ('push', 'tag_push'):
-            raise ValidationError("Unsupported GitLab event type")
-
-        sha = request.data.get('checkout_sha')
-        if kind == 'tag_push':
-            if sha:
-                # When a commit SHA is present, we need to add a reference on an existing commit
-                self.update_revision_references(repo, sha)
-
-            else:
-                # If there isn't any SHA it means that a tag was deleted
-                ref = request.data.get('ref')
-                if not ref:
-                    raise ValidationError('Missing tag reference')
-
-                # Delete existing tag
-                tag_name = ref[10:] if ref.startswith('refs/tags/') else ref
-                repo.refs.filter(name=tag_name, type=GitRefType.Tag).delete()
-
-            return
-
-        if not sha:
-            # If there isn't any SHA it means that a branch was deleted
-            ref = request.data.get('ref')
-            if not ref:
-                raise ValidationError('Missing branch reference')
-
-            # Delete existing branch
-            branch_name = ref[11:] if ref.startswith('refs/heads/') else ref
-            repo.refs.filter(name=branch_name, type=GitRefType.Branch).delete()
-
-            return
-
-        # Already took care of this event
-        if repo.revisions.filter(hash=sha).exists():
-            return
-
-        rev = self.create_revision(repo, sha)
-
-        self.start_imports(rev)
-
-
-git_providers = [
-    GitLabProvider,
-]
-oauth_to_git = {
-    "gitlab": GitLabProvider,
-}
-
-
-def get_provider(name):
-    return next(filter(lambda p: p.__name__ == name, git_providers), None)
-
-
-def from_oauth(name):
-    return oauth_to_git.get(name)

arkindex/process/serializers/git.py

@@ -58,12 +58,3 @@ class RevisionWithRefsSerializer(serializers.ModelSerializer):
             'commit_url',
             'refs',
         )
-
-
-class ExternalRepositorySerializer(serializers.Serializer):
-    """
-    Serialize a Git repository from an external API
-    """
-    id = serializers.IntegerField(min_value=0)
-    name = serializers.CharField(read_only=True)
-    url = serializers.URLField(read_only=True)

arkindex/process/serializers/workers.py

-import base64
-import urllib
-import uuid
 from collections import defaultdict
 from enum import Enum
 from textwrap import dedent
@@ -8,7 +5,6 @@ from textwrap import dedent
 from django.core.exceptions import ValidationError as DjangoValidationError
 from django.db import transaction
 from django.db.models import Max, Q
-from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError
 
@@ -390,29 +386,9 @@ class RepositorySerializer(serializers.ModelSerializer):
     """
     Serialize a repository
     """
-    enabled = serializers.BooleanField(read_only=True)
-    git_clone_url = serializers.SerializerMethodField()
     workers = WorkerLightSerializer(many=True, read_only=True)
     authorized_users = serializers.SerializerMethodField(read_only=True)
 
-    @extend_schema_field(serializers.CharField(allow_null=True))
-    def get_git_clone_url(self, repository):
-        # This check avoid to set git_clone_url when this serializer is used to list multiple
-        # repositories because self.instance value would be a list, even with Ponos task authentication.
-        # We also restrict to enabled repositories as disabled repos do not have OAuth credentials,
-        # and restrict to the repository set on the task's process, as it should only clone that one.
-        process = get_process_from_task_auth(self.context['request'])
-        if (
-            process
-            and isinstance(self.instance, Repository)
-            and self.instance.enabled
-            and process.revision_id is not None
-            and process.revision.repo_id == self.instance.id
-        ):
-            url = urllib.parse.urlparse(self.instance.url)
-            return f"https://oauth2:{repository.credentials.token}@{url.netloc}{url.path}"
-        return None
-
     def get_authorized_users(self, repo) -> int:
         count = getattr(repo, 'authorized_users', None)
         if count is None:
@@ -424,8 +400,6 @@ class RepositorySerializer(serializers.ModelSerializer):
         fields = (
             'id',
             'url',
-            'enabled',
-            'git_clone_url',
             'workers',
             'authorized_users',
         )
@@ -686,11 +660,7 @@ class DockerWorkerVersionSerializer(serializers.ModelSerializer):
         """
         # Retrieve or create the Git repository
         repository, created_repo = Repository.objects.using('default').get_or_create(
-            url=validated_data['repository_url'],
-            defaults={
-                # Generate a default hook token (DB constraint) even if no webhook is created
-                'hook_token': base64.b64encode(uuid.uuid4().bytes).decode('utf-8')
-            },
+            url=validated_data['repository_url']
         )
         # Grant an admin access to the repository in case it got created
         if created_repo:

arkindex/process/tests/test_create_process.py

@@ -12,6 +12,7 @@ from arkindex.process.models import (
     ActivityState,
     Process,
     ProcessMode,
+    Repository,
     WorkerActivity,
     WorkerVersion,
     WorkerVersionState,
@@ -41,8 +42,7 @@ class TestCreateProcess(FixtureAPITestCase):
         cls.private_ml_class = cls.private_corpus.ml_classes.create(name='chouquette')
 
         # Workers ProcessMode
-        cls.creds = cls.user.credentials.get()
-        cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+        cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
         cls.rev_1 = cls.repo.revisions.get()
         cls.rev_2 = cls.repo.revisions.create(
             hash='2',

arkindex/process/tests/test_docker_worker_version.py

@@ -17,8 +17,7 @@ class TestDockerWorkerVersion(FixtureAPITestCase):
     @classmethod
     def setUpTestData(cls):
         super().setUpTestData()
-        cls.creds = cls.user.credentials.get()
-        cls.repo = cls.creds.repos.get(url='http://my_repo.fake/workers/worker')
+        cls.repo = Repository.objects.get(url='http://my_repo.fake/workers/worker')
         cls.rev = cls.repo.revisions.get()
         cls.worker = Worker.objects.get(slug='reco')
         cls.version = cls.worker.versions.get()