From 291f9d45f16b6e957363bf9e3229439e573cf22b Mon Sep 17 00:00:00 2001
From: Yoann Schneider <yschneider@teklia.com>
Date: Mon, 13 May 2024 09:30:03 +0000
Subject: [PATCH] New documentation structure
---
.pre-commit-config.yaml | 2 +-
.../{configure.md => configure/index.md} | 8 +-
.../usage.md => implem/configure/secrets.md} | 10 +-
.../user_configuration/bool_config.png | Bin
.../user_configuration/configuration_form.png | Bin
.../user_configuration/dict_config.png | Bin
.../user_configuration/enum_config.png | Bin
.../user_configuration/float_config.png | Bin
.../user_configuration/integer_config.png | Bin
.../user_configuration/list_config.png | Bin
.../user_configuration/model_config.png | Bin
.../user_configuration/string_config.png | Bin
.../{workers => implem/configure}/yaml.md | 12 --
docs/contents/implem/index.md | 4 +-
docs/contents/implem/process/artifacts.md | 5 +
docs/contents/implem/process/helpers.md | 5 +
docs/contents/implem/process/index.md | 5 +
docs/contents/implem/run/docker.md | 5 +
docs/contents/implem/run/index.md | 5 +
.../run-local.md => implem/run/local.md} | 2 +-
.../ci/index.md => implem/test/ci.md} | 10 +-
docs/contents/implem/test/index.md | 5 +
docs/contents/implem/test/unit.md | 5 +
docs/contents/{workers => init}/create.md | 5 +-
docs/contents/init/index.md | 5 +
.../contents/{workers => init}/maintenance.md | 0
.../{workers => init}/template-structure.md | 6 +-
docs/contents/pick/base.md | 5 +
docs/contents/pick/dataset.md | 5 +
docs/contents/pick/element.md | 5 +
docs/contents/pick/index.md | 5 +
docs/contents/secrets/index.md | 5 -
docs/contents/ship/arkindex.md | 5 +
docs/contents/ship/index.md | 5 +
.../contents/{workers => ship}/publication.md | 2 +-
docs/contents/tutorials/arkindex_model.md | 5 +
docs/contents/tutorials/artifacts.md | 5 +
docs/contents/tutorials/base.md | 5 +
docs/contents/tutorials/dataset.md | 5 +
docs/contents/tutorials/elements.md | 5 +
docs/contents/tutorials/hugging_face.md | 5 +
docs/contents/tutorials/index.md | 5 +
docs/contents/tutorials/training.md | 5 +
docs/contents/workers/index.md | 16 --
docs/contributing.md | 195 ++++++++++++++++++
docs/dev/build_docs.md | 24 ---
mkdocs.yml | 56 +++--
47 files changed, 369 insertions(+), 98 deletions(-)
rename docs/contents/implem/{configure.md => configure/index.md} (91%)
rename docs/contents/{secrets/usage.md => implem/configure/secrets.md} (75%)
rename docs/contents/{workers => implem/configure}/user_configuration/bool_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/configuration_form.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/dict_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/enum_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/float_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/integer_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/list_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/model_config.png (100%)
rename docs/contents/{workers => implem/configure}/user_configuration/string_config.png (100%)
rename docs/contents/{workers => implem/configure}/yaml.md (96%)
create mode 100644 docs/contents/implem/process/artifacts.md
create mode 100644 docs/contents/implem/process/helpers.md
create mode 100644 docs/contents/implem/process/index.md
create mode 100644 docs/contents/implem/run/docker.md
create mode 100644 docs/contents/implem/run/index.md
rename docs/contents/{workers/run-local.md => implem/run/local.md} (99%)
rename docs/contents/{workers/ci/index.md => implem/test/ci.md} (91%)
create mode 100644 docs/contents/implem/test/index.md
create mode 100644 docs/contents/implem/test/unit.md
rename docs/contents/{workers => init}/create.md (97%)
create mode 100644 docs/contents/init/index.md
rename docs/contents/{workers => init}/maintenance.md (100%)
rename docs/contents/{workers => init}/template-structure.md (92%)
create mode 100644 docs/contents/pick/base.md
create mode 100644 docs/contents/pick/dataset.md
create mode 100644 docs/contents/pick/element.md
create mode 100644 docs/contents/pick/index.md
delete mode 100644 docs/contents/secrets/index.md
create mode 100644 docs/contents/ship/arkindex.md
create mode 100644 docs/contents/ship/index.md
rename docs/contents/{workers => ship}/publication.md (83%)
create mode 100644 docs/contents/tutorials/arkindex_model.md
create mode 100644 docs/contents/tutorials/artifacts.md
create mode 100644 docs/contents/tutorials/base.md
create mode 100644 docs/contents/tutorials/dataset.md
create mode 100644 docs/contents/tutorials/elements.md
create mode 100644 docs/contents/tutorials/hugging_face.md
create mode 100644 docs/contents/tutorials/index.md
create mode 100644 docs/contents/tutorials/training.md
delete mode 100644 docs/contents/workers/index.md
create mode 100644 docs/contributing.md
delete mode 100644 docs/dev/build_docs.md
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f49b4884..670e16ee 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -38,7 +38,7 @@ repos:
rev: v0.39.0
hooks:
- id: markdownlint
- args: ['--disable', 'MD013', 'MD046', '--']
+ args: ['--fix', '--disable', 'MD013', 'MD046', '--']
- repo: meta
hooks:
- id: check-useless-excludes
diff --git a/docs/contents/implem/configure.md b/docs/contents/implem/configure/index.md
similarity index 91%
rename from docs/contents/implem/configure.md
rename to docs/contents/implem/configure/index.md
index 493a8a16..4b988c17 100644
--- a/docs/contents/implem/configure.md
+++ b/docs/contents/implem/configure/index.md
@@ -71,9 +71,9 @@ This step shows that there are a lot of sources for the actual configuration tha
The convention is to always give the final word to the user. This means that when the user configuration is filled, its values must be the last to override the worker's `config` attribute. If a model configuration was set, its values must override this attribute before the user configuration's.
-The worker configuration may specify default values for some parameters (see [this section](../workers/yaml.md#setting-up-user-configurable-parameters) for more details about worker configuration). These default values are stored in the `user_configuration` dictionary attribute.
+The worker configuration may specify default values for some parameters (see [this section](yaml.md#setting-up-user-configurable-parameters) for more details about worker configuration). These default values are stored in the `user_configuration` dictionary attribute.
-This is also when the secrets (see [this section](../secrets/usage.md#declaring-secrets-in-workers) to learn more about secrets) are actually downloaded. They are stored in the `secrets` dictionary attribute.
+This is also when the secrets (see [this section](secrets.md#declaring-secrets-in-workers) to learn more about secrets) are actually downloaded. They are stored in the `secrets` dictionary attribute.
An Arkindex-mode exclusive step is done after all that: the cache setup. Some workers benefit a lot, performance-wise, from having a SQLite cache artifact from previous workers. This is mostly used in processes with multiple workers with dependencies, where the second worker needs the results of the first one to work. The database is initialized, the tables created and its version checked as it must match the one supported by the Arkindex instances. The database is then merged with any other database generated by previous worker runs.
@@ -81,9 +81,9 @@ Once all information is retrieved and stored in the worker, the configuration is
## Developer mode
-In the developer mode, the worker execution is not linked to anything on Arkindex. Therefore, the only configuration the worker can use is provided via the `--config` CLI argument. It supports YAML-formatted file and it should be similar to the `configuration` section of the [worker configuration file](../workers/yaml.md#single-worker-configuration), without the `user_configuration` details. More details about how to create the local worker configuration are available in [this section](../workers/run-local.md).
+In the developer mode, the worker execution is not linked to anything on Arkindex. Therefore, the only configuration the worker can use is provided via the `--config` CLI argument. It supports YAML-formatted file and it should be similar to the `configuration` section of the [worker configuration file](yaml.md#single-worker-configuration), without the `user_configuration` details. More details about how to create the local worker configuration are available in [this section](../run/local.md).
-The multiple configuration sources from the Arkindex-mode are merged into a unique one here. The configuration parameters are parsed as well as the list of required secrets. The secrets are loaded using a local Arkindex client. Again, see the [section about local execution](../workers/run-local.md) for more details.
+The multiple configuration sources from the Arkindex-mode are merged into a unique one here. The configuration parameters are parsed as well as the list of required secrets. The secrets are loaded using a local Arkindex client. Again, see the [section about local execution](../run/local.md) for more details.
One information cannot be retrieved directly from the configuration file and is required in some cases: the ID of the Arkindex corpus which the elements processed belong to. This is retrieved via the `ARKINDEX_CORPUS_ID` environment variable.
diff --git a/docs/contents/secrets/usage.md b/docs/contents/implem/configure/secrets.md
similarity index 75%
rename from docs/contents/secrets/usage.md
rename to docs/contents/implem/configure/secrets.md
index d9050be5..2778bba7 100644
--- a/docs/contents/secrets/usage.md
+++ b/docs/contents/implem/configure/secrets.md
@@ -1,4 +1,10 @@
-# Usage
+# Use secrets in your worker
+
+## Overview
+
+Secrets are text payloads shared securely between the Arkindex instance and any worker. It is generally used to store sensitive values that may give Arkindex users access to any resources that should be private, for example because they cost money or are proprietary.
+
+For more information about secrets, please visit [the Arkindex documentation](https://doc.arkindex.org/secrets/).
## Accessing secrets in the API
@@ -35,7 +41,7 @@ missing from the Arkindex instance.
```
For more information on the `.arkindex.yml` file,
-see [YAML configuration](../workers/yaml.md).
+see [YAML configuration](yaml.md).
## Accessing secrets in Python code
diff --git a/docs/contents/workers/user_configuration/bool_config.png b/docs/contents/implem/configure/user_configuration/bool_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/bool_config.png
rename to docs/contents/implem/configure/user_configuration/bool_config.png
diff --git a/docs/contents/workers/user_configuration/configuration_form.png b/docs/contents/implem/configure/user_configuration/configuration_form.png
similarity index 100%
rename from docs/contents/workers/user_configuration/configuration_form.png
rename to docs/contents/implem/configure/user_configuration/configuration_form.png
diff --git a/docs/contents/workers/user_configuration/dict_config.png b/docs/contents/implem/configure/user_configuration/dict_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/dict_config.png
rename to docs/contents/implem/configure/user_configuration/dict_config.png
diff --git a/docs/contents/workers/user_configuration/enum_config.png b/docs/contents/implem/configure/user_configuration/enum_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/enum_config.png
rename to docs/contents/implem/configure/user_configuration/enum_config.png
diff --git a/docs/contents/workers/user_configuration/float_config.png b/docs/contents/implem/configure/user_configuration/float_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/float_config.png
rename to docs/contents/implem/configure/user_configuration/float_config.png
diff --git a/docs/contents/workers/user_configuration/integer_config.png b/docs/contents/implem/configure/user_configuration/integer_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/integer_config.png
rename to docs/contents/implem/configure/user_configuration/integer_config.png
diff --git a/docs/contents/workers/user_configuration/list_config.png b/docs/contents/implem/configure/user_configuration/list_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/list_config.png
rename to docs/contents/implem/configure/user_configuration/list_config.png
diff --git a/docs/contents/workers/user_configuration/model_config.png b/docs/contents/implem/configure/user_configuration/model_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/model_config.png
rename to docs/contents/implem/configure/user_configuration/model_config.png
diff --git a/docs/contents/workers/user_configuration/string_config.png b/docs/contents/implem/configure/user_configuration/string_config.png
similarity index 100%
rename from docs/contents/workers/user_configuration/string_config.png
rename to docs/contents/implem/configure/user_configuration/string_config.png
diff --git a/docs/contents/workers/yaml.md b/docs/contents/implem/configure/yaml.md
similarity index 96%
rename from docs/contents/workers/yaml.md
rename to docs/contents/implem/configure/yaml.md
index 245b9202..054f2f4b 100644
--- a/docs/contents/workers/yaml.md
+++ b/docs/contents/implem/configure/yaml.md
@@ -86,18 +86,6 @@ include:
`docker`
: Regroups Docker-related configuration attributes:
- <!--
- TODO: Make the path relative to the YAML file itself, in the case of a
- separate file for a single worker?
- https://gitlab.teklia.com/arkindex/tasks/-/issues/95
- -->
- <!--
- TODO: Implement this!
- https://gitlab.teklia.com/arkindex/tasks/-/issues/93
- `image`: Tag of an existing Docker image to use for this worker instead of building a
- custom image from a Dockerfile.
- -->
-
- `build`
: Path towards a Dockerfile used to build this worker, relative to the root of
the repository. Defaults to `Dockerfile`.
diff --git a/docs/contents/implem/index.md b/docs/contents/implem/index.md
index 4764155d..a8fec187 100644
--- a/docs/contents/implem/index.md
+++ b/docs/contents/implem/index.md
@@ -1,10 +1,10 @@
-# Worker Implementation
+# Implement your worker
This section presents
- the different stages happening during a worker execution:
- the initialization
- - the [configuration](./configure.md)
+ - the [configuration](configure/index.md)
- the execution
- the conception of a worker
- the architecture
diff --git a/docs/contents/implem/process/artifacts.md b/docs/contents/implem/process/artifacts.md
new file mode 100644
index 00000000..7df7fc0d
--- /dev/null
+++ b/docs/contents/implem/process/artifacts.md
@@ -0,0 +1,5 @@
+# Artifacts
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/implem/process/helpers.md b/docs/contents/implem/process/helpers.md
new file mode 100644
index 00000000..95c286ad
--- /dev/null
+++ b/docs/contents/implem/process/helpers.md
@@ -0,0 +1,5 @@
+# Arkindex API Helpers
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/implem/process/index.md b/docs/contents/implem/process/index.md
new file mode 100644
index 00000000..dae7c578
--- /dev/null
+++ b/docs/contents/implem/process/index.md
@@ -0,0 +1,5 @@
+# Processing
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/implem/run/docker.md b/docs/contents/implem/run/docker.md
new file mode 100644
index 00000000..c911d4a5
--- /dev/null
+++ b/docs/contents/implem/run/docker.md
@@ -0,0 +1,5 @@
+# Run your worker in Docker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/implem/run/index.md b/docs/contents/implem/run/index.md
new file mode 100644
index 00000000..b4ea3438
--- /dev/null
+++ b/docs/contents/implem/run/index.md
@@ -0,0 +1,5 @@
+# Run your worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/workers/run-local.md b/docs/contents/implem/run/local.md
similarity index 99%
rename from docs/contents/workers/run-local.md
rename to docs/contents/implem/run/local.md
index 7a0b8520..fc0e87b1 100644
--- a/docs/contents/workers/run-local.md
+++ b/docs/contents/implem/run/local.md
@@ -1,4 +1,4 @@
-# Running your worker locally
+# Run your worker locally
Once you have implemented a worker, you can run it on some Arkindex elements on your own machine to test it.
diff --git a/docs/contents/workers/ci/index.md b/docs/contents/implem/test/ci.md
similarity index 91%
rename from docs/contents/workers/ci/index.md
rename to docs/contents/implem/test/ci.md
index 812832ef..8d2c5bd1 100644
--- a/docs/contents/workers/ci/index.md
+++ b/docs/contents/implem/test/ci.md
@@ -3,9 +3,6 @@
This page describes how continuous integration (CI) is used in workers created
using the `base-worker` template.
-For more information on creating workers, see
-[Setting up a worker](../create.md).
-
## Default template
When creating a worker with our official template, a `.gitlab-ci.yml` file has
@@ -45,8 +42,7 @@ project and validate various rules:
- Check your YAML files are well formatted
- Fix some common spelling errors
-You can set up pre-commit to run locally too; see
-[Activating the pre-commit hook](../create.md#activating-the-pre-commit-hook).
+You can set up pre-commit to run locally too; see Activating the pre-commit hook.
## Testing
@@ -59,10 +55,6 @@ allowing you to check the validity of your code before merging it.
Unit tests allow you to prevent regressions in your code when making changes,
and find bugs before they make their way into production.
-<!-- TODO:
-For more information, see [Writing unit tests for your worker](../tests).
--->
-
## Building
When the `test` & `lint` jobs run successfully, the `docker` job runs. It will
diff --git a/docs/contents/implem/test/index.md b/docs/contents/implem/test/index.md
new file mode 100644
index 00000000..f93dfb7e
--- /dev/null
+++ b/docs/contents/implem/test/index.md
@@ -0,0 +1,5 @@
+# Tests
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/implem/test/unit.md b/docs/contents/implem/test/unit.md
new file mode 100644
index 00000000..c8844358
--- /dev/null
+++ b/docs/contents/implem/test/unit.md
@@ -0,0 +1,5 @@
+# Unit tests
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/workers/create.md b/docs/contents/init/create.md
similarity index 97%
rename from docs/contents/workers/create.md
rename to docs/contents/init/create.md
index def1c1b3..f77cef75 100644
--- a/docs/contents/workers/create.md
+++ b/docs/contents/init/create.md
@@ -96,8 +96,7 @@ template and making it available on a GitLab instance.
For a worker to be accessible from an Arkindex instance, it needs to be sent
to a repository on a GitLab project. A GitLab project will also allow you to
-manage different versions of a worker and run
-[automated checks](ci/index.md) on your code.
+manage different versions of a worker and run [automated checks](../implem/test/ci.md) on your code.
#### To create a GitLab project
@@ -211,7 +210,7 @@ This section assumes you have Maintainer or Owner access to the GitLab project.
4. Open your GitLab project in a browser.
-5. Click on the blue icon indicating that [CI](ci/index.md)
+5. Click on the blue icon indicating that [CI](../implem/test/ci.md)
is running on your repository, and wait for it to turn green to confirm
everything worked.
diff --git a/docs/contents/init/index.md b/docs/contents/init/index.md
new file mode 100644
index 00000000..03813974
--- /dev/null
+++ b/docs/contents/init/index.md
@@ -0,0 +1,5 @@
+# Initialize your worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/workers/maintenance.md b/docs/contents/init/maintenance.md
similarity index 100%
rename from docs/contents/workers/maintenance.md
rename to docs/contents/init/maintenance.md
diff --git a/docs/contents/workers/template-structure.md b/docs/contents/init/template-structure.md
similarity index 92%
rename from docs/contents/workers/template-structure.md
rename to docs/contents/init/template-structure.md
index 141fab3d..de418d32 100644
--- a/docs/contents/workers/template-structure.md
+++ b/docs/contents/init/template-structure.md
@@ -7,7 +7,7 @@ package, a Docker build, with the best development practices:
`.arkindex.yml`
: YAML configuration file that allows Arkindex to understand what it should do
with this repository.
- To learn more about this file, see [YAML configuration](yaml.md).
+ To learn more about this file, see [YAML configuration](../implem/configure/yaml.md).
`.cookiecutter.yaml`
: YAML file that stores the options you defined when creating a new worker.
@@ -28,7 +28,7 @@ package, a Docker build, with the best development practices:
`.gitlab-ci.yml`
: Configures the GitLab CI jobs and pipelines.
To learn more about the configuration we provide, see
- [GitLab Continuous Integration for workers](ci/index.md).
+ [GitLab Continuous Integration for workers](../implem/test/ci.md).
`.isort.cfg`
: Configures the automatic Python import sorting rules.
@@ -50,7 +50,7 @@ package, a Docker build, with the best development practices:
: Configures the worker's Python package.
`ci/build.sh`
-: Script that gets run by [CI](ci/index.md) pipelines
+: Script that gets run by [CI](../implem/test/ci.md) pipelines
to build the Docker image.
`tests/test_worker.py`
diff --git a/docs/contents/pick/base.md b/docs/contents/pick/base.md
new file mode 100644
index 00000000..f24fe14f
--- /dev/null
+++ b/docs/contents/pick/base.md
@@ -0,0 +1,5 @@
+# Base Worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/pick/dataset.md b/docs/contents/pick/dataset.md
new file mode 100644
index 00000000..b81b3663
--- /dev/null
+++ b/docs/contents/pick/dataset.md
@@ -0,0 +1,5 @@
+# Dataset worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/pick/element.md b/docs/contents/pick/element.md
new file mode 100644
index 00000000..65a74eea
--- /dev/null
+++ b/docs/contents/pick/element.md
@@ -0,0 +1,5 @@
+# Element worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/pick/index.md b/docs/contents/pick/index.md
new file mode 100644
index 00000000..beaf4422
--- /dev/null
+++ b/docs/contents/pick/index.md
@@ -0,0 +1,5 @@
+# Pick your worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/secrets/index.md b/docs/contents/secrets/index.md
deleted file mode 100644
index 37c0a75a..00000000
--- a/docs/contents/secrets/index.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Secrets
-
-Secrets are text payloads shared securely between the Arkindex instance and any worker. It is generally used to store sensitive values that may give Arkindex users access to any resources that should be private, for example because they cost money or are proprietary.
-
-For more information about secrets, please visit [the Arkindex documentation](https://doc.arkindex.org/secrets/).
diff --git a/docs/contents/ship/arkindex.md b/docs/contents/ship/arkindex.md
new file mode 100644
index 00000000..d9d61d3e
--- /dev/null
+++ b/docs/contents/ship/arkindex.md
@@ -0,0 +1,5 @@
+# Arkindex execution
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/ship/index.md b/docs/contents/ship/index.md
new file mode 100644
index 00000000..7a98ee9b
--- /dev/null
+++ b/docs/contents/ship/index.md
@@ -0,0 +1,5 @@
+# Ship your worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/workers/publication.md b/docs/contents/ship/publication.md
similarity index 83%
rename from docs/contents/workers/publication.md
rename to docs/contents/ship/publication.md
index e8f400fd..a5cc6364 100644
--- a/docs/contents/workers/publication.md
+++ b/docs/contents/ship/publication.md
@@ -2,7 +2,7 @@
## Publication process
-While workers may be [run locally](./run-local.md), they are usually written to launch on an Arkindex instance. A worker version on Arkindex is basically a configuration and a URL towards a Docker image, used at runtime during processes. The worker [template](./template-structure.md) includes an automatic publication process towards any Arkindex instance.
+While workers may be [run locally](../implem/run/local.md), they are usually written to launch on an Arkindex instance. A worker version on Arkindex is basically a configuration and a URL towards a Docker image, used at runtime during processes. The worker [template](../init/template-structure.md) includes an automatic publication process towards any Arkindex instance.
## How it works
diff --git a/docs/contents/tutorials/arkindex_model.md b/docs/contents/tutorials/arkindex_model.md
new file mode 100644
index 00000000..d274b249
--- /dev/null
+++ b/docs/contents/tutorials/arkindex_model.md
@@ -0,0 +1,5 @@
+# Arkindex model version
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/artifacts.md b/docs/contents/tutorials/artifacts.md
new file mode 100644
index 00000000..cb6ee487
--- /dev/null
+++ b/docs/contents/tutorials/artifacts.md
@@ -0,0 +1,5 @@
+# Use artifacts
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/base.md b/docs/contents/tutorials/base.md
new file mode 100644
index 00000000..5badedf6
--- /dev/null
+++ b/docs/contents/tutorials/base.md
@@ -0,0 +1,5 @@
+# Base worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/dataset.md b/docs/contents/tutorials/dataset.md
new file mode 100644
index 00000000..b81b3663
--- /dev/null
+++ b/docs/contents/tutorials/dataset.md
@@ -0,0 +1,5 @@
+# Dataset worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/elements.md b/docs/contents/tutorials/elements.md
new file mode 100644
index 00000000..65a74eea
--- /dev/null
+++ b/docs/contents/tutorials/elements.md
@@ -0,0 +1,5 @@
+# Element worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/hugging_face.md b/docs/contents/tutorials/hugging_face.md
new file mode 100644
index 00000000..567bdcae
--- /dev/null
+++ b/docs/contents/tutorials/hugging_face.md
@@ -0,0 +1,5 @@
+# Hugging Face
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/index.md b/docs/contents/tutorials/index.md
new file mode 100644
index 00000000..c206860c
--- /dev/null
+++ b/docs/contents/tutorials/index.md
@@ -0,0 +1,5 @@
+# Tutorials
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/tutorials/training.md b/docs/contents/tutorials/training.md
new file mode 100644
index 00000000..fcbeec54
--- /dev/null
+++ b/docs/contents/tutorials/training.md
@@ -0,0 +1,5 @@
+# Training worker
+
+!!! warning
+
+ This section is under construction.
diff --git a/docs/contents/workers/index.md b/docs/contents/workers/index.md
deleted file mode 100644
index 7dd4e534..00000000
--- a/docs/contents/workers/index.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# Workers
-
-Arkindex has a powerful system to run asynchronous tasks. Those are based on
-Docker images, and can do about anything (ML processing, but also to import
-data into Arkindex, export from Arkindex to another system or file format...)
-
-This section consists of the following guides:
-
-## Contents
-
-* [Setting up a new worker](create.md)
-* [Running your worker locally](run-local.md)
-* [Maintaining a worker](maintenance.md)
-* [GitLab CI for workers](ci/index.md)
-* [YAML configuration](yaml.md)
-* [Template structure](template-structure.md)
diff --git a/docs/contributing.md b/docs/contributing.md
new file mode 100644
index 00000000..edc700a4
--- /dev/null
+++ b/docs/contributing.md
@@ -0,0 +1,195 @@
+# Contributing to Base Worker
+
+We welcome every contribution possible to improve Base Worker. Whether it is Python code, translations, design suggestions, small fixes, etc.
+
+If you want to discuss some ideas, send us an email at `team (at) teklia.com`!
+
+## Cloning the source code
+
+It is as simple as running the following command:
+
+```bash
+git clone git@gitlab.teklia.com:workers/base-worker.git
+```
+
+As Base Worker is open-source, you do not need any authentication to retrieve its source code. However, if you wish to work on Base Worker source code and submit patches, you will need to authenticate yourself. Steps to do so are detailed just below.
+
+## GitLab setup
+
+We assume you already have:
+
+- a working [git](https://git-scm.com/) client installation on your computer
+- an SSH key to authenticate yourself on a GitLab instance
+
+You can find more information [here](https://docs.gitlab.com/ee/user/ssh.html).
+
+### Sign up on Teklia's GitLab instance
+
+At Teklia, we use [GitLab](https://gitlab.teklia.com/) **a lot**. We use it to host all our source code, work as a team, build and ship our projects, etc.
+
+You can sign up on Teklia's GitLab instance [here](https://gitlab.teklia.com/users/sign_up).
+
+Please note that your account will need to be approved by an administrator which could take up to one business day.
+
+### Add your SSH key
+
+Once you are connected to [gitlab.teklia.com](https://gitlab.teklia.com/), you need to add your **SSH public key** to your user account on Teklia's GitLab instance. To do so, you can follow [the official GitLab instructions](https://docs.gitlab.com/ee/user/ssh.html#add-an-ssh-key-to-your-gitlab-account).
+
+!!! warning "Security warning"
+
+ **Do not** use HTTPS to clone a repository, it will require you to type your GitLab password when cloning/pushing.
+
+### Activate 2FA
+
+To gain and keep access on Base Worker's GitLab repository as a contributor, your GitLab account must be protected with Multiple Factor Authentication (generally using one time passwords generated from your phone).
+
+You will need to setup an app like [Google Authenticator](https://support.google.com/accounts/answer/1066447?hl=fr&co=GENIE.Platform%3DAndroid) or [Duo Mobile](https://duo.com/product/multi-factor-authentication-mfa/duo-mobile-app) on your phone, and [then follow these steps](https://docs.gitlab.com/ee/user/profile/account/two_factor_authentication.html#enabling-2fa).
+
+If you do not complete the 2FA initial activation within 48 hours of creating your account, it will be automatically disabled.
+
+## Submitting a patch to Base Worker's source code
+
+### Find issues
+
+We use GitLab to track issues (either bugs, or new features).
+
+You can look at Base Worker's issues from different views:
+
+- through the current [milestone](https://gitlab.teklia.com/workers/base-worker/-/milestones), the most important issues are listed there, as they are the most time-sensitive
+- through the [full issue list](https://gitlab.teklia.com/workers/base-worker/-/issues)
+
+!!! info
+
+ Please ask first if you can work on an issue, by simply commenting on it. Be aware that some issues may be way more complex than others.
+
+#### Priorities
+
+Issues should have a priority label:
+
+- **P1** are high priority, usually bugs badly affecting users experience,
+- **P2** are at the normal priority level
+- **P3** are low priority
+
+If nothing is set, assume **P2** by default.
+
+### Work on a branch
+
+To start working on a patch, you must first create a **git branch**, based on the current `master` branch:
+
+```bash
+# Start from master
+git checkout master
+
+# Update master to latest available revision
+git pull origin master
+
+# Create a new branch
+git checkout -b my-new-branch
+```
+
+Each new commit will then be stored on that new branch named `my-new-branch`.
+
+To name your branch, use a name:
+
+1. in English,
+2. in lowercase,
+3. without spaces, use dashes `-` to link words,
+4. explicit and related to your current work.
+
+Here are some examples of suitable branch names:
+
+- `remove-model-x` when the goal is to remove a python module named `X`
+- `fix-invalid-chars-search` when the patch fixes a **bug** related to search
+- `bump-dependency-y`
+- `add-super-feature`
+- `feature-z`
+
+Please avoid:
+
+- naming your branch with the issue ID, we prefer explicit naming here,
+- using another language than English,
+- spaces, camelCase, underscores, etc.
+
+### Publish your work
+
+You are allowed to push directly on Base Worker's repository, except for the `master` branch. The goal is for your code to reach `master` once the following steps are completed:
+
+1. unit tests are all OK, meaning that all jobs in the CI stage named `test` ended in success
+2. formatting has been validated by a tool, meaning that all jobs in the CI stage named `test` ended in success
+3. the code itself has been approved by a human reviewer
+
+To get approvals, you need to [create a Merge Request (also called MR)](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html) on GitLab.
+
+When pushing [your code from your local branch](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html#when-you-use-git-commands-locally), you will notice some output in the console with a link towards `gitlab.teklia.com`: it will allow you to create a Merge Request (or view the previously created one) in 2 clicks.
+
+Once your work is ready, configure your Merge Request as follows:
+
+- Assign yourself as the **Assignee**.
+- Assign [@babadie](https://gitlab.teklia.com/babadie) as the **Reviewer**, he will either review or re-assign to another reviewer when needed.
+- Set an explicit name, in English, properly formatted.
+- Add a reference to the issue you are working on in the description:
+ - `Closes #XYZ` if you fully solve the mentioned issue,
+ - `Ref #XYZ` if you only want to link your Merge Request to the issue.
+- Fill in the description with an explanation of the changes you made.
+- If the related issue has a milestone set, set the same milestone on this Merge Request.
+
+If you are not confident the work being published is yet ready for review, you can prefix your Merge Request name with `Draft:`; that will tell the reviewer to wait a bit before diving into your code. Do not forget to remove that prefix once your code is ready.
+
+The reviewer may leave some comments directly on the Merge Request, asking you for updates. Please resolve all of them (or discuss them if you disagree), publish some commits fixing the issues, and then [ask for a new review](https://docs.gitlab.com/ee/user/project/merge_requests/reviews/#request-a-new-review). Rinse and repeat until the reviewer approves and merges your code!
+
+### Update your branch
+
+As other developers are working on Base Worker, sometimes features/bugfixes/etc will land on `master` while your Merge Request touches the same parts. You may get conflicts here and will need to solve them using a **rebase**.
+
+!!! info
+
+ It is your responsibility, as a developer, to maintain your code in a **mergeable state**: no conflicts and up to date with the latest `master`.
+
+To update using **rebase**, while working on your branch:
+
+```bash
+# Retrieve the latest updates from master
+git fetch origin master
+
+# Now the remote reference to origin/master has been updated, you can rebase on top of it
+# Be aware that the local reference to master is not yet updated, it is only updated with git pull
+git rebase origin/master
+```
+
+The `git rebase` operation will ask you to solve manually conflicts (if any). Please [follow this guide](https://www.atlassian.com/git/tutorials/rewriting-history/git-rebase) or ask us for help if you are lost.
+
+### Landing your code
+
+If you have reached this step, congratulations and many thanks! Your code has been approved and should already be merged, as that is the responsibility of the reviewer.
+
+Your work will be shipped in [the next Base Worker release](https://workers.arkindex.org/releases/) along with latest features, bugfixes and other contributions.
+
+### Building the documentation
+
+#### Setup
+
+Add the `docs` extra when installing `arkindex-base-worker`:
+
+```sh
+# In a clone of the Git repository
+pip install .[docs]
+```
+
+Build the documentation using `mkdocs serve -v`. The documentation should be available at <http://localhost:8000>.
+
+#### Writing documentation
+
+This documentation uses [Sphinx](http://www.sphinx-doc.org/) and was generated using [mkdocs](https://mkdocs.org/) and [mkdocstrings](https://mkdocstrings.github.io/).
+
+#### Linting
+
+This documentation is subject to linting using `markdownlint`, integrated into
+`pre-commit`. You can run it locally by typing `pre-commit run`. You should use
+`pre-commit install` to have the pre-commit hook run before each Git commit.
+
+The linting rules that `markdownlint` applies can be found on [its documentation](https://github.com/igorshubovych/markdownlint-cli/blob/master/README.md).
+
+Some rules are disabled:
+
+- [MD013](https://github.com/DavidAnson/markdownlint/blob/main/doc/md013.md): Line length
+- [MD046](https://github.com/DavidAnson/markdownlint/blob/main/doc/md046.md): Code block style
diff --git a/docs/dev/build_docs.md b/docs/dev/build_docs.md
deleted file mode 100644
index 2d6207c7..00000000
--- a/docs/dev/build_docs.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# Building the documentation
-
-## Setup
-
-Add the `docs` extra when installing `arkindex-base-worker`:
-
-```sh
-# In a clone of the Git repository
-pip install .[docs]
-```
-
-Build the documentation using `mkdocs serve -v`. The documentation should be available as <http://localhost:8000>
-
-## Writing documentation
-
-This documentation uses [Sphinx](http://www.sphinx-doc.org/) and was generated using [mkdocs](https://mkdocs.org/) and [mkdocstrings](https://mkdocstrings.github.io/).
-
-## Linting
-
-This documentation is subject to linting using `markdownlint`, integrated into
-`pre-commit`. You can run it locally by typing `pre-commit run`. You should use
-`pre-commit install` to have the pre-commit hook run before each Git commit.
-
-The linting rules that `markdownlint` applies can be found on [its documentation](https://github.com/igorshubovych/markdownlint-cli/blob/master/README.md).
diff --git a/mkdocs.yml b/mkdocs.yml
index 6180b8cd..41c780ac 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -60,21 +60,48 @@ plugins:
nav:
- Home: index.md
- - How to create a worker:
- - contents/workers/index.md
- - Setting up a new worker: contents/workers/create.md
- - Running your worker locally: contents/workers/run-local.md
- - Publishing your worker on Arkindex: contents/workers/publication.md
- - Maintaining a worker: contents/workers/maintenance.md
- - GitLab CI for workers: contents/workers/ci/index.md
- - YAML configuration: contents/workers/yaml.md
- - Template structure: contents/workers/template-structure.md
- - Using secrets in workers:
- - contents/secrets/index.md
- - Usage: contents/secrets/usage.md
- - Worker Implementation:
+ - Pick your worker:
+ - contents/pick/index.md
+ - Base Worker: contents/pick/base.md
+ - Element Worker: contents/pick/element.md
+ - Dataset Worker: contents/pick/dataset.md
+ - Initialize your worker:
+ - contents/init/index.md
+ - Setting up a new worker: contents/init/create.md
+ - Template structure: contents/init/template-structure.md
+ - Maintaining a worker: contents/init/maintenance.md
+ - Implement your worker:
- contents/implem/index.md
- - Configuration: contents/implem/configure.md
+ - Configuration:
+ - contents/implem/configure/index.md
+ - YAML configuration: contents/implem/configure/yaml.md
+ - Use secrets in your worker: contents/implem/configure/secrets.md
+ - Processing:
+ - contents/implem/process/index.md
+ - Arkindex API Helpers: contents/implem/process/helpers.md
+ - Artifacts: contents/implem/process/artifacts.md
+ - Tests:
+ - contents/implem/test/index.md
+ - Unit tests: contents/implem/test/unit.md
+ - Gitlab CI: contents/implem/test/ci.md
+ - Run your worker:
+ - contents/implem/run/index.md
+ - Run your worker locally: contents/implem/run/local.md
+ - Run your worker in Docker: contents/implem/run/docker.md
+ - Ship your worker:
+ - contents/ship/index.md
+ - Arkindex publication: contents/ship/publication.md
+ - Arkindex execution: contents/ship/arkindex.md
+ - Tutorials:
+ - contents/tutorials/index.md
+ - Element worker: contents/tutorials/elements.md
+ - Base worker: contents/tutorials/base.md
+ - Dataset worker: contents/tutorials/dataset.md
+ - Hugging Face: contents/tutorials/hugging_face.md
+ - Arkindex model version: contents/tutorials/arkindex_model.md
+ - Training worker: contents/tutorials/training.md
+ - Use artifacts: contents/tutorials/artifacts.md
+ - Contributing: contributing.md
- Python Reference:
- Base Worker: ref/base_worker.md
- Arkindex Workers: ref/arkindex_workers.md
@@ -94,7 +121,6 @@ nav:
- Cache: ref/cache.md
- Utils: ref/utils.md
- Releases: releases.md
- - Documentation development: dev/build_docs.md
markdown_extensions:
- smarty
--
GitLab