Merge branch 'pull-nerval-from-pypi' into 'main'

Depend on Nerval pypi package

See merge request !353

.gitlab-ci.yml

@@ -3,10 +3,6 @@ stages:
   - build
   - deploy
 
-variables:
-  # Submodule clone
-  GIT_SUBMODULE_STRATEGY: recursive
-
 lint:
   image: python:3.10
   stage: test
@@ -46,6 +42,14 @@ test:
     - apt-get update -q
     - apt-get install -q -y libgl1
 
+    # Add netrc file
+    - |
+      echo "
+      machine gitlab.teklia.com
+      login gitlab-ci-token
+      password $CI_JOB_TOKEN
+      " > ~/.netrc
+
   except:
     - schedules
 
@@ -64,6 +68,15 @@ docker-build:
   except:
     - schedules
 
+  before_script:
+    # Add netrc file
+    - |
+      echo "
+      machine gitlab.teklia.com
+      login gitlab-ci-token
+      password $CI_JOB_TOKEN
+      " > ~/.netrc
+
   script:
     - ci/build.sh
 
@@ -75,7 +88,15 @@ docker-build:
       - public
 
   before_script:
-    - pip install -e .[docs]
+    # Add netrc file
+    - |
+      echo "
+      machine gitlab.teklia.com
+      login gitlab-ci-token
+      password $CI_JOB_TOKEN
+      " > ~/.netrc
+
+    - pip install --index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple -e .[docs]
 
   script:
     - mkdocs build --strict --verbose

.gitmodules

-[submodule "nerval"]
-	path = nerval
-	url = ../../ner/nerval.git

Dockerfile

@@ -7,12 +7,10 @@ RUN apt-get -y update && \
 
 WORKDIR /src
 
-# Copy submodule data
-COPY nerval nerval
-
 # Copy DAN data
 COPY dan dan
 COPY requirements.txt *-requirements.txt setup.py VERSION README.md ./
 
-# Install DAN as a package
-RUN pip install . --no-cache-dir
+# Install DAN as a package with GitLab package registry
+RUN --mount=type=secret,id=netrc,target=/root/.netrc \
+    pip install . --no-cache-dir --index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple

README.md

@@ -10,9 +10,20 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 For development and tests purpose it may be useful to install the project as a editable package with pip.
 
-- Use a virtualenv (e.g. with virtualenvwrapper `mkvirtualenv -a . dan`)
-- Initialize the [`Nerval`](https://gitlab.teklia.com/ner/nerval) submodule (e.g. `git submodule update --init --recursive`)
-- Install `dan` as a package (e.g. `pip install -e .`)
+This package is based on a GitLab package registry containing all the nerval source code.
+You need [a personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) and access to the [nerval repository](https://gitlab.teklia.com/ner/nerval) in order to install this module. You will need to add the below to your `~/.netrc` file:
+
+```shell
+machine gitlab.teklia.com
+login __token__
+password <YOUR_PERSONAL_TOKEN>
+```
+
+Then you can install the package as a editable package with pip:
+
+```shell
+pip3 install --index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple -e .
+```
 
 ### Linter
 

ci/build.sh

@@ -17,7 +17,7 @@ fi
 IMAGE_TAG="$CI_REGISTRY_IMAGE:$VERSION"
 
 cd $CI_PROJECT_DIR
-docker build -f Dockerfile . -t "$IMAGE_TAG"
+docker build -f Dockerfile . -t "$IMAGE_TAG" --secret id=netrc,src=$HOME/.netrc
 
 # Publish the image on the main branch or on a tag
 if [ "$CI_COMMIT_REF_NAME" = "main" -o -n "$CI_COMMIT_TAG" ]; then

dan/ocr/evaluate.py

@@ -15,6 +15,9 @@ import numpy as np
 import torch
 import torch.multiprocessing as mp
 from edlib import align, getNiceAlignment
+from nerval.evaluate import evaluate
+from nerval.parse import parse_bio
+from nerval.utils import print_results
 from prettytable import MARKDOWN, PrettyTable
 
 from dan.bio import convert
@@ -22,9 +25,6 @@ from dan.ocr.manager.metrics import Inference
 from dan.ocr.manager.training import Manager
 from dan.ocr.utils import add_metrics_table_row, create_metrics_table, update_config
 from dan.utils import parse_tokens, read_json
-from nerval.evaluate import evaluate
-from nerval.parse import parse_bio
-from nerval.utils import print_results
 
 logger = logging.getLogger(__name__)
 

docs/get_started/development.md

@@ -62,9 +62,11 @@ Add the `docs` extra when installing `teklia-dan`:
 
 ```shell
 # In a clone of the Git repository
-pip install .[docs]
+pip install --index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple .[docs]
 ```
 
+The `--index-url` argument is required to find the `nerval` package.
+
 Build the documentation using `mkdocs serve -v`. You can then write in [Markdown](https://www.markdownguide.org/) in the relevant `docs/*.md` files, and see live output on http://localhost:8000.
 
 ### Linter

docs/get_started/index.md

@@ -2,41 +2,43 @@
 
 ## Installation
 
-To use DAN in your own environment, you need to install it as a dependency or manually.
+DAN is based on a GitLab package registry containing all the nerval source code.
+You need [a personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) and access to the [nerval repository](https://gitlab.teklia.com/ner/nerval) in order to install this module. You will need to add the below to your `~/.netrc` file:
+
+```shell
+machine gitlab.teklia.com
+login __token__
+password <YOUR_PERSONAL_TOKEN>
+```
+
+If you install DAN as a dependency, the host must have access to this configuration file to be able to download the [nerval repository](https://gitlab.teklia.com/ner/nerval) package.
 
 ### As a dependency
 
 To install DAN as a dependency, you need to first add the following line to your `requirements.txt` file:
 
 ```shell
-teklia-dan @ git+ssh://git@gitlab.teklia.com/atr/dan.git
+--index-url https://gitlab.teklia.com/api/v4/projects/98/packages/pypi/simple --extra-index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple
+teklia-dan
 ```
 
+The `--index-url` argument is required to find the `DAN` package, the `--extra-index-url` argument is needed to find the `nerval` dependency.
+
 Then you can install it via pip:
 
 ```shell
 pip install -r requirements.txt
 ```
 
-### Manually
+### Directly
 
-To install DAN manually, you need to first clone via:
+To install DAN directly, you can install it via pip:
 
 ```shell
-git clone git@gitlab.teklia.com:atr/dan.git
+pip3 install --index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple --extra-index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple teklia-dan
 ```
 
-Then you can initialize the [`Nerval`](https://gitlab.teklia.com/ner/nerval) submodule:
-
-```shell
-git submodule update --init --recursive
-```
-
-Then you can install it via pip:
-
-```shell
-pip install .
-```
+The `--index-url` argument is required to find the `DAN` package, the `--extra-index-url` argument is needed to find the `nerval` dependency.
 
 ---
 

docs/usage/train/config.md

@@ -242,9 +242,11 @@ To log your experiment on MLFlow, you need to:
 - install the extra requirements via
 
 ```shell
-$ pip install .[mlflow]
+$ pip install --index-url https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple .[mlflow]
 ```
 
+The `--index-url` argument is required to find the `nerval` package.
+
 - update the following arguments:
 
 | Name                           | Description                             | Type  | Default |

nerval @ 525c1a9e

-Subproject commit 525c1a9e6d5a33075669085148247e2604dd092f

pyproject.toml

@@ -24,6 +24,7 @@ known-third-party = [
     "cv2",
     "editdistance",
     "imageio",
+    "nerval",
     "numpy",
     "scipy",
     "tensorboard",

requirements.txt

--e ./nerval
 albumentations==1.3.1
 arkindex-export==0.1.9
 boto3==1.26.124
@@ -13,6 +12,7 @@ PyYAML==6.0
 scipy==1.10.1
 sentencepiece==0.1.99
 teklia-line-image-extractor==0.2.8rc5
+teklia-nerval==0.3.1
 tenacity==8.2.3
 tensorboard==2.12.2
 torch==2.0.0

tox.ini

@@ -14,5 +14,8 @@ deps =
 commands =
     pytest {tty:--color=yes} {posargs}
 
+setenv =
+  PIP_INDEX_URL=https://gitlab.teklia.com/api/v4/projects/210/packages/pypi/simple
+
 [pytest]
 testpaths= tests