diff --git a/.gitignore b/.gitignore
index c5908ea7619719461231968e8412ff604380ba31..098ada102b7ba6de43ba20bfbceb3fc4bc69bb07 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,10 +83,14 @@ kk/
laia/version.py
# data
-datasets/
+^datasets/
test-resources/
# benchmarks
benchmarks/basic
benchmarks/distributed
benchmarks/half
+
+# JS
+node_modules/
+package-lock.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8b4db1cb498f394f54c284153a981f0838971563..402ff46ef9300a6f4f1738b47f7ff41c847c4dce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -112,16 +112,16 @@ bump-python-deps:
# Make sure docs still build correctly
.docs:
- image: python:3.10
+ image: node:20
artifacts:
paths:
- public
before_script:
- - pip install -e .[docs]
+ - npm install
script:
- - mkdocs build --strict --verbose
+ - npx antora antora-playbook.yml --html-url-extension-style=indexify
docs-build:
extends: .docs
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index dea8c88b13ed48ffa2ae2554f28e3a8ace853573..3b0394fe11875acfe81f6fd166547d9f2ca45599 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -120,7 +120,5 @@ Enhancement suggestions are tracked as [GitLab issues](https://gitlab.teklia.com
- Run the test suite (including a coverage report), through `tox`
### Improving The Documentation
-- Create a virtual environment, you can use [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) or [conda](https://docs.conda.io/en/latest/).
-- Install dependencies, through `pip install ".[doc]"`.
-- Visualize the current documentation, through `mkdocs serve -v`.
-- Make sure it builds without warning, through `mkdocs build --strict`.
+- Install dependencies, through `npm install`.
+- Visualize the current documentation, through `make docs`.
diff --git a/Makefile b/Makefile
index f9322fd1664aa6f32a3259456bb19a2e285859f3..9c7b04b703eff036a106bea8539a2ea4bb21c577 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,7 @@
-.PHONY: release
+.PHONY: release docs
+
+docs:
+ npx antora antora-playbook.yml
release:
# Grep the version from pyproject.toml, squeeze multiple spaces, delete double and single quotes, get 3rd val.
diff --git a/antora-playbook.yml b/antora-playbook.yml
new file mode 100644
index 0000000000000000000000000000000000000000..28f2e2e5fc74942c05fd4e9a4f33bd8e64f54976
--- /dev/null
+++ b/antora-playbook.yml
@@ -0,0 +1,21 @@
+site:
+ title: PyLaia
+ start_page: pylaia::index.adoc
+content:
+ sources:
+ - url: .
+ branches: HEAD
+ start_path: docs
+ui:
+ bundle:
+ url: https://gitlab.com/antora/antora-ui-default/-/jobs/artifacts/HEAD/raw/build/ui-bundle.zip?job=bundle-stable
+ snapshot: true
+ supplemental_files: ./docs/ui
+
+
+antora:
+ extensions:
+ - '@antora/lunr-extension'
+
+output:
+ dir: ./public
diff --git a/docs/antora.yml b/docs/antora.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ba9d333ba570d64be38bd2f16e2b103e247a2a73
--- /dev/null
+++ b/docs/antora.yml
@@ -0,0 +1,5 @@
+name: pylaia
+version: ~
+title: PyLaia
+nav:
+- modules/ROOT/nav.adoc
diff --git a/docs/assets/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f.jpg b/docs/assets/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f.jpg
deleted file mode 100644
index 842a440c763f27636ab7927415f2113e9432d95a..0000000000000000000000000000000000000000
Binary files a/docs/assets/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f.jpg and /dev/null differ
diff --git a/docs/assets/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4.jpg b/docs/assets/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4.jpg
deleted file mode 100644
index 6be7c26391adf7975c8ba2390f8c39a4dee38bf6..0000000000000000000000000000000000000000
Binary files a/docs/assets/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4.jpg and /dev/null differ
diff --git a/docs/get_started/development.md b/docs/get_started/development.md
deleted file mode 100644
index a6f551b7a2a103c323f5be4443f5a3d33ba5f49c..0000000000000000000000000000000000000000
--- a/docs/get_started/development.md
+++ /dev/null
@@ -1,40 +0,0 @@
-# Development
-
-PyLaia uses different tools during its development.
-
-## Linter
-
-Code syntax is analyzed before submitting the code.
-
-To run the linter tools suite you may use [pre-commit](https://pre-commit.com).
-
-```shell
-pip install pre-commit
-pre-commit run -a
-```
-
-## Tests
-
-### Unit tests
-
-Tests are executed using [tox](https://tox.wiki/en/latest/).
-
-```shell
-pip install .[test]
-tox
-```
-
-## Documentation
-
-This documentation uses [Sphinx](http://www.sphinx-doc.org/) and was generated using [MkDocs](https://mkdocs.org/) and [mkdocstrings](https://mkdocstrings.github.io/).
-
-### Setup
-
-Add the `docs` extra when installing `pylaia`:
-
-```shell
-# In a clone of the Git repository
-pip install .[docs]
-```
-
-Build the documentation using `mkdocs serve -v`. You can then write in [Markdown](https://www.markdownguide.org/) in the relevant `docs/*.md` files, and see live output on http://localhost:8000.
diff --git a/docs/get_started/index.md b/docs/get_started/index.md
deleted file mode 100644
index 12c52bc1c50c59d976ed276faa98a9a1c0ce9dd1..0000000000000000000000000000000000000000
--- a/docs/get_started/index.md
+++ /dev/null
@@ -1,32 +0,0 @@
-# Installation
-
-To use PyLaia in your own environment, you need to install from PyPi or manually.
-
-## From PyPi
-
-To install PyLaia from [PyPi](https://pypi.org/project/pylaia/), use this command:
-
-```shell
-pip install pylaia
-```
-
-## From source
-
-To install PyLaia manually, you need to first clone via:
-
-```shell
-git clone git@gitlab.teklia.com:atr/pylaia.git
-```
-
-Then you can install it via pip:
-
-```shell
-pip install .
-```
-
----
-
-Get started with:
-
-- [Development](development.md)
-- [Usage](../usage/index.md)
diff --git a/docs/index.md b/docs/index.md
deleted file mode 100644
index 2f5b4aace12d735e1302945173f2ddcd259da624..0000000000000000000000000000000000000000
--- a/docs/index.md
+++ /dev/null
@@ -1,22 +0,0 @@
-# PyLaia
-
-## What is PyLaia?
-
-PyLaia is a toolkit for Automatic Text Recognition (ATR) and Keyword Spotting (KWS).
-
-PyLaia is flexible, open-source, device-agnostic, and can be used to express a wide variety of experiments, including (but not limited to) training and inference over Convolutional and Recurrent based deep Neural Network models.
-The software is extensible and easily configurable and provides a rich set of functional layers with a particular focus on ATR.
-
-## History
-
-PyLaia is the successor of [Laia](https://github.com/jpuigcerver/Laia). It was developed by 3 members ([@jpuigcerver](https://github.com/jpuigcerver), [@mauvilsa](https://github.com/mauvilsa), [@dmartinalbo](https://github.com/dmartinalbo)) of the Pattern Recognition and Human Language Technology (PRHLT) research center in 2016.
-
-The toolkit was originally developed using Torch. When Torch's development was discontinued in 2017, it became clear that building PyLaia as a second-generation system using PyTorch as its foundation was a logical step. PyLaia was written in 2018 by [@jpuigcerver](https://github.com/jpuigcerver) as a Ph.D. thesis experiment and by [@carmocca](https://github.com/carmocca) as an undergraduate final project.
-
-Since 2022, three members of [TEKLIA](https://teklia.com/) ([@babadie](https://gitlab.teklia.com/babadie), [@yschneider](https://gitlab.teklia.com/yschneider), [@starride](https://gitlab.teklia.com/starride)) maintain and improve the toolkit.
-
-## Get started
-
-Click [here](original_paper.md) to learn more about the original paper.
-
-[Get started with PyLaia](get_started/index.md) now!
diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..09db71be8ca704af4093fc4fd09f93285c23a27d
--- /dev/null
+++ b/docs/modules/ROOT/nav.adoc
@@ -0,0 +1,15 @@
+* xref:original_paper.adoc[Original paper]
+* Get started
+** xref:get_started/index.adoc[]
+** xref:get_started/development.adoc[Development]
+* Usage
+** xref:usage/index.adoc[]
+** Dataset
+*** xref:usage/datasets/index.adoc[]
+*** xref:usage/datasets/format.adoc[Dataset formatting]
+** xref:usage/initialization/index.adoc[Model initialization]
+** xref:usage/training/index.adoc[Training]
+** xref:usage/prediction/index.adoc[Prediction]
+** xref:usage/netout/index.adoc[Netout]
+** xref:usage/language_models/index.adoc[Explicit language modeling]
+* xref:releases.adoc[Releases]
diff --git a/docs/modules/ROOT/pages/get_started/development.adoc b/docs/modules/ROOT/pages/get_started/development.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..61ff2e388ef97731c88abd8031cd9f8b27b6a3ac
--- /dev/null
+++ b/docs/modules/ROOT/pages/get_started/development.adoc
@@ -0,0 +1,49 @@
+[#development]
+= Development
+
+PyLaia uses different tools during its development.
+
+[#linter]
+== Linter
+
+Code syntax is analyzed before submitting the code.
+
+To run the linter tools suite you may use https://pre-commit.com[pre-commit].
+
+[,shell]
+----
+pip install pre-commit
+pre-commit run -a
+----
+
+[#tests]
+== Tests
+
+[#unit-tests]
+=== Unit tests
+
+Tests are executed using https://tox.wiki/en/latest/[tox].
+
+[,shell]
+----
+pip install .[test]
+tox
+----
+
+[#documentation]
+== Documentation
+
+This documentation uses http://www.sphinx-doc.org/[Sphinx] and was generated using https://mkdocs.org/[MkDocs] and https://mkdocstrings.github.io/[mkdocstrings].
+
+[#setup]
+=== Setup
+
+Add the `docs` extra when installing `pylaia`:
+
+[,shell]
+----
+# In a clone of the Git repository
+pip install .[docs]
+----
+
+Build the documentation using `mkdocs serve -v`. You can then write in https://www.markdownguide.org/[Markdown] in the relevant `docs/*.md` files, and see live output on http://localhost:8000.
diff --git a/docs/modules/ROOT/pages/get_started/index.adoc b/docs/modules/ROOT/pages/get_started/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..55e7ffeacbddc7f4a4404bd12b950648f676e647
--- /dev/null
+++ b/docs/modules/ROOT/pages/get_started/index.adoc
@@ -0,0 +1,38 @@
+[#installation]
+= Installation
+
+To use PyLaia in your own environment, you need to install from PyPi or manually.
+
+[#from-pypi]
+== From PyPi
+
+To install PyLaia from https://pypi.org/project/pylaia/[PyPi], use this command:
+
+[,shell]
+----
+pip install pylaia
+----
+
+[#from-source]
+== From source
+
+To install PyLaia manually, you need to first clone via:
+
+[,shell]
+----
+git clone git@gitlab.teklia.com:atr/pylaia.git
+----
+
+Then you can install it via pip:
+
+[,shell]
+----
+pip install .
+----
+
+'''
+
+Get started with:
+
+* xref:./development.adoc[Development]
+* xref:usage/index.adoc[Usage]
diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..12269ef85edf5ae51795564c8d68c60220e9d5d5
--- /dev/null
+++ b/docs/modules/ROOT/pages/index.adoc
@@ -0,0 +1,26 @@
+[#pylaia]
+= PyLaia
+
+[#what-is-pylaia]
+== What is PyLaia?
+
+PyLaia is a toolkit for Automatic Text Recognition (ATR) and Keyword Spotting (KWS).
+
+PyLaia is flexible, open-source, device-agnostic, and can be used to express a wide variety of experiments, including (but not limited to) training and inference over Convolutional and Recurrent based deep Neural Network models.
+The software is extensible and easily configurable and provides a rich set of functional layers with a particular focus on ATR.
+
+[#history]
+== History
+
+PyLaia is the successor of https://github.com/jpuigcerver/Laia[Laia]. It was developed by 3 members (https://github.com/jpuigcerver[@jpuigcerver], https://github.com/mauvilsa[@mauvilsa], https://github.com/dmartinalbo[@dmartinalbo]) of the Pattern Recognition and Human Language Technology (PRHLT) research center in 2016.
+
+The toolkit was originally developed using Torch. When Torch's development was discontinued in 2017, it became clear that building PyLaia as a second-generation system using PyTorch as its foundation was a logical step. PyLaia was written in 2018 by https://github.com/jpuigcerver[@jpuigcerver] as a Ph.D. thesis experiment and by https://github.com/carmocca[@carmocca] as an undergraduate final project.
+
+Since 2022, three members of https://teklia.com/[TEKLIA] (https://gitlab.teklia.com/babadie[@babadie], https://gitlab.teklia.com/yschneider[@yschneider], https://gitlab.teklia.com/starride[@starride]) maintain and improve the toolkit.
+
+[#get-started]
+== Get started
+
+Click xref:./original_paper.adoc[here] to learn more about the original paper.
+
+xref:./get_started/index.adoc[Get started with PyLaia] now!
diff --git a/docs/modules/ROOT/pages/original_paper.adoc b/docs/modules/ROOT/pages/original_paper.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..516802f7bb8f285700a3069db14971d88cba4a5e
--- /dev/null
+++ b/docs/modules/ROOT/pages/original_paper.adoc
@@ -0,0 +1,21 @@
+[#original-paper]
+= Original paper
+
+The original PyLaia model was presented in the paper entitled: https://ieeexplore.ieee.org/document/8269951[_Are Multidimensional Recurrent Layers Really Necessary for Handwritten Text Recognition?_ from Joan Puigcerver, published in the 14th IAPR International Conference on Document Analysis and Recognition (ICDAR 2017)].
+
+The full text is available on this http://www.jpuigcerver.net/pubs/jpuigcerver_icdar2017.pdf[page].
+
+Recommended citation:
+
+[,bibtex]
+----
+@INPROCEEDINGS{PyLaia,
+ author={Puigcerver, Joan},
+ booktitle={2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR)},
+ title={Are Multidimensional Recurrent Layers Really Necessary for Handwritten Text Recognition?},
+ year={2017},
+ volume={01},
+ number={},
+ pages={67-72},
+ doi={10.1109/ICDAR.2017.20}}
+----
diff --git a/docs/modules/ROOT/pages/releases.adoc b/docs/modules/ROOT/pages/releases.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..7999f025d79136adb84101ce5f84b5a4471c86cc
--- /dev/null
+++ b/docs/modules/ROOT/pages/releases.adoc
@@ -0,0 +1,250 @@
+[#releases]
+= Releases
+
+[#1-1-1]
+== 1.1.1
+
+Released on *12 August 2024* • View on https://gitlab.teklia.com/atr/pylaia/-/releases/1.1.1[Gitlab]
+
+[#breaking-changes-1-1-1]
+=== Breaking changes
+
+* The https://gitlab.teklia.com/atr/nnutils/[nnutils] library is no longer maintained and is only compatible with Python 3.6, 3.7, 3.8. As such its dependency has been removed. The `crnn.use_masks` parameter has been removed. It is still supported to keep the compatibility with older training configuration but will be ignored.
+
+[#features-1-1-1]
+=== Feature
+
+* The number of worker processes created in dataloaders is now exposed through the `data.num_workers` parameter.
+* There is a new command to run basic checks and compute statistics on your training dataset. Learn more about it in https://atr.pages.teklia.com/pylaia/usage/datasets/[the documentation].
+* Pretraining is now available. Load the weights of a previous checkpoint using the `train.pretrain` parameter when fine-tuning a model on a new dataset. Learn more about it in https://atr.pages.teklia.com/pylaia/usage/training/#resume-training-from-a-checkpoint[the documentation].
+* When training on a small dataset, freezing some of the layers can help with model convergence. The `train.freeze_layers` parameter supports freezing:
+ ** convolutional layers,
+ ** recurrent layers,
+ ** linear layers.
+* Proper support for right-to-left (RTL) languages is now available. Enable it using the `data.reading_order` argument both during https://atr.pages.teklia.com/pylaia/usage/training/#train-on-right-to-left-reading-order[training] and https://atr.pages.teklia.com/pylaia/usage/prediction/#predict-on-right-to-left-data[decoding].
+
+[#dependencies-1-1-1]
+=== Dependencies
+
+* Bumped https://pypi.org/project/pytorch-lightning/[pytorch-lightning] to version `1.4.2`.
+* Bumped https://pypi.org/project/textdistance/[textdistance] to version `4.6.1`.
+
+[#misc]
+=== Misc
+
+* A deprecation warning from jsonargparse was fixed.
+* The package's metadata are now stored in `pyproject.toml` as per https://peps.python.org/pep-0621/[PEP-0621].
+* PyLaia now uses https://docs.astral.sh/ruff/[ruff] for linting and formatting.
+
+[#1-1-0]
+== 1.1.0
+
+Released on *22 December 2023* • View on https://gitlab.teklia.com/atr/pylaia/-/releases/1.1.0[Gitlab]
+
+[#breaking-changes-1-1-0]
+=== Breaking changes
+
+* Official support for Python3.8 has been dropped. This doesn't mean that the current code doesn't run on python3.8, we simply do not test that compatibility anymore. This decision was made since active support of python 3.8 has stopped for a while now and many libraries in the ML world have stopped supporting it as well.
+
+[#feature-1-1-0]
+=== Feature
+
+* A Docker image with the needed code to use this library is now built on every tag.
+* The coverage of our tests suite is displayed again as a GitLab badge on the repository as well as in the README.md file.
+
+[#documentation-1-1-10]
+=== Documentation
+
+* Many sections were added to the documentation:
+ ** for the https://atr.pages.teklia.com/pylaia/usage/initialization/[pylaia-htr-create-model] command,
+ ** for https://atr.pages.teklia.com/pylaia/usage/datasets/[dataset formatting],
+ ** for the https://atr.pages.teklia.com/pylaia/usage/training/[pylaia-htr-train-ctc] command and https://atr.pages.teklia.com/pylaia/usage/training/#resume-training-from-a-checkpoint[fine-tuning],
+ ** for the https://atr.pages.teklia.com/pylaia/usage/prediction/[pylaia-htr-decode-ctc] command,
+ ** for the https://atr.pages.teklia.com/pylaia/usage/netout/[pylaia-htr-netout] command,
+ ** to https://atr.pages.teklia.com/pylaia/usage/language_models/[train] https://kheafield.com/code/kenlm/[KenLM] language models,
+ ** the full Python code reference.
+* A contribution guide and a code of conduct were added for new contributors.
+
+[#dependencies-1-1-0]
+=== Dependencies
+
+* Bumped https://pypi.org/project/pytorch-lightning/[pytorch-lightning] to version `1.3.0`
+* Some dependencies were pinned to a version to avoid breakage:
+ ** https://pypi.org/project/natsort/[natsort] was pinned to version `8.4.0`,
+ ** https://pypi.org/project/textdistance/[textdistance] was pinned to version `4.6.0`,
+ ** https://pypi.org/project/scipy/[scipy] was pinned to version `1.11.3`,
+ ** https://pypi.org/project/matplotlib/[matplotlib] was pinned to version `3.8.2`,
+ ** https://pypi.org/project/numpy/[numpy] direct dependency was removed since it's installed through `scipy` and `matplotlib`.
+* PyLaia dropped support for python 3.8 so the https://pypi.org/project/dataclasses/[dataclasses] dependency was dropped.
+
+[#misc-1-1-0]
+=== Misc
+
+* The `torch.testing.assert_allclose` has been replaced by `torch.testing.assert_close` since it became deprecated in https://github.com/pytorch/pytorch/issues/61844[PyTorch 1.12.0].
+
+[#1-0-7]
+== 1.0.7
+
+Released on *18 October 2023* • View on https://gitlab.teklia.com/atr/pylaia/-/releases/1.0.7[Gitlab]
+
+[#feature-1-0-7]
+=== Feature
+
+* When using a language model, a confidence score is now returned based on the log-likelyhood of the hypothesis.
+
+[#documentation-1-0-7]
+=== Documentation
+
+A public documentation is now available on https://atr.pages.teklia.com/pylaia/. It's still under construction but next releases will add more and more content.
+
+[#dependencies-1-0-7]
+=== Dependencies
+
+* Bumped https://pypi.org/project/pytorch-lightning/[pytorch-lightning] to version `1.1.7`
+* Bumped GitHub action https://github.com/codecov/codecov-action[codecov/codecov-action] to version `3`
+* Bumped GitHub action https://github.com/actions/setup-python[actions/setup-python] to version `4`
+* Bumped GitHub action https://github.com/actions/checkout[actions/checkout] to version `4`
+
+[#development-1-0-7]
+=== Development
+
+* Releases are now built more easily through a Makefile.
+* The documentation is also redeployed after each push on `master` branch.
+* Fixed a test that behaved differently locally and during CI.
+
+[#1-0-6]
+== 1.0.6
+
+Released on *12 September 2023* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.6[Github]
+
+[#feature-1-0-6]
+=== Feature
+
+* During training, too small images are now padded to be able to pass the multiple convolution layers.
+
+[#documentation-1-0-6]
+=== Documentation
+
+* Fixed typos.
+
+[#dependencies-1-0-6]
+=== Dependencies
+
+* Replaced https://pillow.readthedocs.io/en/stable/releasenotes/2.7.0.html#antialias-renamed-to-lanczos[deprecated Pillow resampling method] `Image.ANTIALIAS` to `Image.Resample.Lanczos`.
+
+[#development-1-0-6]
+=== Development
+
+* Pre-commit hooks were updated.
+
+[#1-0-5]
+== 1.0.5
+
+Released on *29 March 2023* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.5[Github]
+
+[#dependencies-1-0-5]
+=== Dependencies
+
+* Requires `torch` version `1.13.0` or `1.13.1`.
+* Requires `torchvision` version `0.14.0` or `0.14.1` (depending on `torch` version).
+
+[#1-0-4]
+== 1.0.4
+
+Released on *4 January 2023* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.4[Github]
+
+[#dependencies-1-0-4]
+=== Dependencies
+
+* Requires `torch` version `1.13.0`.
+
+[#1-0-3]
+== 1.0.3
+
+Released on *12 December 2022* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.3[Github]
+
+[#feature-1-0-3]
+=== Feature
+
+* Now able to decode using a trained Language model through beam search decoding.
+* Exposes https://pytorch.org/docs/stable/data.html#multi-process-data-loading[torch Dataloaders's num_workers] parameter on the Python training function to limit resource usage when needed.
+
+[#dependencies-1-0-3]
+=== Dependencies
+
+* Added dependency to `torchaudio` version `0.13.0`.
+
+[#development-1-0-3]
+=== Development
+
+* Package version is now tracked through the `VERSION` file.
+
+[#1-0-2]
+== 1.0.2
+
+Released on *7 December 2022* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.2[Github]
+
+[#dependencies-1-0-2]
+=== Dependencies
+
+* Pinned dependency to `pytorch-lightning` to version `1.1.0`.
+
+[#1-0-1]
+== 1.0.1
+
+Released on *7 December 2022* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.1[Github]
+
+[#1-0-0]
+== 1.0.0
+
+Released on *2 December 2020* • View on https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.0[Github]
+
+[#added-1-0-0]
+=== Added
+
+* Support distributed training
+* Scripts can now be configured using yaml configuration files
+* Add support for the SGD and Adam optimizers
+* Support color images
+* Log the installed version of each module when scripts are called from shell
+* Add char/word segmentation to the decode script
+* Add several badges to the README
+* Support using a `ReduceLROnPlateau` scheduler during training
+* A CSV file (metrics.csv) is now created with the results obtained during training
+* Add CONTRIBUTING file
+* Training now can include GPU stats in the progress bar
+* Add isort to pre-commit to keep consistent imports throughout the codebase
+* Users can run the PyLaia scripts using Python now
+* Support half-precision training for fixed height models.
+* Add script to visualize the segmentation output
+* Use Codecov to produce test coverage reports
+* Code is now analyzed using CodeFactor
+
+[#changed-1-0-0]
+=== Changed
+
+* Make Python 3.6 the minimum supported version
+* Make PyTorch 1.4.0 the minimum supported version
+* Remove `ImageToTensor` in favor of vision transform `ToImageTensor`
+* Remove all of the internal logic (`engine`, `actions`, `hooks`, etc) in favor of pytorch-lightning's constructs
+* Change Travis CI for GitHub actions
+* Greatly improve the progress bar. It is used now in all scripts
+* The entire shell API has changed for the better (thanks to jsonargparse). Arguments are now separated into groups and help messages are clearer.
+* Drastically improve our test suite, we now have a 91% coverage
+
+[#removed-1-0-0]
+=== Removed
+
+* Remove egs directory. These live now at https://github.com/carmocca/PyLaia-examples
+* Remove Baidu's CTC loss in favor of PyTorch's
+* Remove PHOC code. Please open an issue if you were using it
+* Remove Dortmund code. Please open an issue if you were using it
+* Remove CTCLatticeGenerator. Please open an issue if you were using it
+* We no longer support saving checkpoints for more than one metric. Will be added back in a future version
+
+[#fixed-1-0-0]
+=== Fixed
+
+* Fix WER calculation when long delimiters are used
+* Exit training if a delimiter is not present in the vocabulary
+* Hundreds of other minor fixes and refactors to improve the code quality!
diff --git a/docs/usage/datasets/format.md b/docs/modules/ROOT/pages/usage/datasets/format.adoc
similarity index 81%
rename from docs/usage/datasets/format.md
rename to docs/modules/ROOT/pages/usage/datasets/format.adoc
index 599e2f0cffdfd684e5a556d448836671cf5444ec..700a97b24fc833e665e736c8f7ee8ef3e751306a 100644
--- a/docs/usage/datasets/format.md
+++ b/docs/modules/ROOT/pages/usage/datasets/format.adoc
@@ -1,9 +1,12 @@
-# Dataset formatting
+[#dataset-formatting]
+= Dataset formatting
To train PyLaia, you need line images and their corresponding transcriptions. The dataset should be divided into three sets: training, validation and test sets.
The dataset should be formatted as follows:
-```bash
+
+[,bash]
+----
# Images
├── images
├── train/
@@ -23,64 +26,78 @@ The dataset should be formatted as follows:
├── test_text.txt
# Symbol list
└── syms.txt
-```
+----
-## Images
+[#images]
+== Images
-By default, images should be resized to a fixed height (recommended value: 128 pixels). This can be done using [ImageMagick's `mogrify`](https://imagemagick.org/script/mogrify.php) function:
-```
-mogrify -resize x128 images/*.jpg
-```
+By default, images should be resized to a fixed height (recommended value: 128 pixels). This can be done using https://imagemagick.org/script/mogrify.php[ImageMagick's `mogrify`] function:
-Note that PyLaia can also support variable size images by setting `--fixed_input_height 0` during [model initialization](../initialization/index.md).
+----
+mogrify -resize x128 images/*.jpg
+----
+Note that PyLaia can also support variable size images by setting `--fixed_input_height 0` during xref:usage/initialization/index.adoc[model initialization].
-## Ground truth
+[#ground-truth]
+== Ground truth
-### Tokenized transcriptions
+[#tokenized-transcriptions]
+=== Tokenized transcriptions
Two files `{train|val}.txt` are required to train the model. They should map image names and tokenized transcriptions for the training and validation sets.
Example:
-```text title="train.txt"
+[,text]
+----
train/im01 f o r <space> d e t <space> t i l f æ l d e <space> d e t <space> s k u l d e <space> l y k k e s <space> D i g
train/im02 a t <space> o p d r i v e <space> d e t <space> o m s k r e v n e <space> e x p l : <space> a f
train/im03 « F r u <space> I n g e r » , <space> a t <space> s e n d e <space> m i g <space> s a m m e
-```
+----
-### Transcriptions
+[#transcriptions]
+=== Transcriptions
Three files `{train|val|test}_text.txt` are required to evaluate your models. They should map image names and non-tokenized transcriptions.
Example:
-```text title="train_text.txt"
+
+[,text]
+----
train/im01 for det tilfælde det skulde lykkes Dig
train/im02 at opdrive det omskrevne expl: af
train/im03 «Fru Inger», at sende mig samme
-```
+----
-### Image list
+[#image-list]
+=== Image list
Three files `{train|val|test}_ids.txt` are required to run predictions. They should list image names without transcriptions and can be obtained with:
-```bash
+
+[,bash]
+----
cut -d' ' -f1 train_text.txt > train_ids.txt
-```
+----
Example:
-```text title="train_ids.txt"
+
+[,text]
+----
train/im01
train/im02
train/im03
-```
+----
-### Symbol list
+[#symbol-list]
+=== Symbol list
Finally, a file named `syms.txt` is required, mapping tokens from the training set and their index, starting with the `<ctc>` token.
Example:
-```text title="syms.txt"
+[,text]
+----
<ctc> 0
! 1
" 2
@@ -193,4 +210,4 @@ z 81
… 109
<unk> 110
<space> 111
-```
+----
diff --git a/docs/usage/datasets/index.md b/docs/modules/ROOT/pages/usage/datasets/index.adoc
similarity index 62%
rename from docs/usage/datasets/index.md
rename to docs/modules/ROOT/pages/usage/datasets/index.adoc
index ac16063ad5d181158af6ed86383d73b22d420cb8..94db2caa8c24ac5b672d3a169d79009272f67a15 100644
--- a/docs/usage/datasets/index.md
+++ b/docs/modules/ROOT/pages/usage/datasets/index.adoc
@@ -1,11 +1,12 @@
-# Dataset
+[#dataset]
+= Dataset
-PyLaia datasets must be formatted following a specific format. Learn how to build a dataset by following this [page](./format.md).
+PyLaia datasets must be formatted following a specific format. Learn how to build a dataset by following this xref:./format.adoc[page].
Once the dataset is created, you may use the `pylaia-htr-dataset-validate` command to compute statistics and make sure your dataset is valid. To know more about the options of this command, use `pylaia-htr-dataset-validate --help`.
-
-## Purpose
+[#purpose]
+== Purpose
This command will:
@@ -19,54 +20,139 @@ If the dataset is valid, the script will:
* display `Dataset is valid` and
* save a summary of the dataset statistics in a Markdown file named after the argument provided in `--statistics_output`.
-## Parameters
+[#parameters]
+== Parameters
The full list of parameters is detailed in this section.
-### General parameters
-
-| Parameter | Description | Type | Default |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------- |
-| `syms` | Positional argument. Path to a file mapping characters to integers. The CTC symbol must be mapped to integer 0. | `str` | |
-| `img_dirs` | Positional argument. Directories containing line images. | `str` | |
-| `tr_txt_table` | Positional argument. Path to a file mapping training image ids and tokenized transcription. | `str` | |
-| `va_txt_table` | Positional argument. Path to a file mapping validation image ids and tokenized transcription. | `str` | |
-| `te_txt_table` | Positional argument. Path to a file mapping validation image ids and tokenized transcription. | `str` | |
-| `fixed_input_height` | Height of the input images. If set to 0, a variable height model will be used (see `adaptive_pooling`). This will be used to compute the model output height at the end of the convolutional layers. | `int` | 0 |
-| `statistics_output` | Where the Markdown summary will be written. | `str` | `"statistics.md"` |
-| `config` | Path to a JSON configuration file | `json` | |
-
-### Common parameters
-
-| Name | Description | Type | Default |
-| ----------------------- | --------------------------------------- | ----- | ------- |
-| `common.train_path` | Directory where the model will be saved | `str` | `.` |
-| `common.model_filename` | Filename of the model. | `str` | `model` |
-
-### Logging arguments
-
-| Name | Description | Type | Default |
-| ------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------- |
-| `logging.fmt` | Logging format. | `str` | `%(asctime)s %(levelname)s %(name)s] %(message)s` |
-| `logging.level` | Logging level. Should be in `{NOTSET,DEBUG,INFO,WARNING,ERROR,CRITICAL}` | `Level` | `INFO` |
-| `logging.filepath` | Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname` | `Optional[str]` | |
-| `logging.overwrite` | Whether to overwrite the logfile or to append. | `bool` | `False` |
-| `logging.to_stderr_level` | If filename is set, use this to log also to stderr at the given level. | `Level` | `ERROR` |
-
-### Train arguments
-
-| Name | Description | Type | Default |
-| ------------------ | ------------------------------------------------- | ------ | ------------- |
-| `train.delimiters` | List of symbols representing the word delimiters. | `List` | `["<space>"]` |
-
-## Examples
+[#general-parameters]
+=== General parameters
+
+|===
+| Parameter | Description | Type | Default
+
+| `syms`
+| Positional argument. Path to a file mapping characters to integers. The CTC symbol must be mapped to integer 0.
+| `str`
+|
+
+| `img_dirs`
+| Positional argument. Directories containing line images.
+| `str`
+|
+
+| `tr_txt_table`
+| Positional argument. Path to a file mapping training image ids and tokenized transcription.
+| `str`
+|
+
+| `va_txt_table`
+| Positional argument. Path to a file mapping validation image ids and tokenized transcription.
+| `str`
+|
+
+| `te_txt_table`
+| Positional argument. Path to a file mapping validation image ids and tokenized transcription.
+| `str`
+|
+
+| `fixed_input_height`
+| Height of the input images. If set to 0, a variable height model will be used (see `adaptive_pooling`). This will be used to compute the model output height at the end of the convolutional layers.
+| `int`
+| 0
+
+| `statistics_output`
+| Where the Markdown summary will be written.
+| `str`
+| `"statistics.md"`
+
+| `config`
+| Path to a JSON configuration file
+| `json`
+|
+|===
+
+[#common-parameters]
+=== Common parameters
+
+|===
+| Name | Description | Type | Default
+
+| `common.train_path`
+| Directory where the model will be saved
+| `str`
+| `.`
+
+| `common.model_filename`
+| Filename of the model.
+| `str`
+| `model`
+|===
+
+[#logging-arguments]
+=== Logging arguments
+
+|===
+| Name | Description | Type | Default
+
+| `logging.fmt`
+| Logging format.
+| `str`
+| `%(asctime)s %(levelname)s %(name)s] %(message)s`
+
+| `logging.level`
+a| Logging level. Should be in
+
+* `NOTSET`
+* `DEBUG`
+* `INFO`
+* `WARNING`
+* `ERROR`
+* `CRITICAL`
+
+| `Level`
+| `INFO`
+
+| `logging.filepath`
+| Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname`
+| `Optional[str]`
+|
+
+| `logging.overwrite`
+| Whether to overwrite the logfile or to append.
+| `bool`
+| `False`
+
+| `logging.to_stderr_level`
+| If filename is set, use this to log also to stderr at the given level.
+| `Level`
+| `ERROR`
+|===
+
+[#train-arguments]
+=== Train arguments
+
+|===
+| Name | Description | Type | Default
+
+| `train.delimiters`
+| List of symbols representing the word delimiters.
+| `List`
+| `["<space>"]`
+|===
+
+[#examples]
+== Examples
This arguments can be passed using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
-### Example with Command Line Arguments (CLI)
+[#example-with-command-line-arguments-cli]
+=== Example with Command Line Arguments (CLI)
Run the following command to create a model:
-```sh
+
+[,sh]
+----
pylaia-htr-dataset-validate /data/Esposalles/dataset/syms.txt \
[/data/Esposalles/dataset/images/] \
/data/Esposalles/dataset/train.txt \
@@ -75,27 +161,33 @@ pylaia-htr-dataset-validate /data/Esposalles/dataset/syms.txt \
--common.experiment_dirname experiment-esposalles/ \
--fixed_input_size 128 \
--statistice_ouput statistics.md
-```
+----
Will output:
-```bash
+
+[,bash]
+----
[2024-04-23 12:58:31,399 INFO laia] Arguments: {'syms': '/data/Esposalles/dataset/syms.txt', 'img_dirs': ['/data/Esposalles/dataset/images/'], 'tr_txt_table': '/data/Esposalles/dataset/train.txt', 'va_txt_table': '/data/Esposalles/dataset/val.txt', 'te_txt_table': '/data/Esposalles/dataset/test.txt', 'fixed_input_height': 128, 'common': CommonArgs(seed=74565, train_path='', model_filename='model', experiment_dirname='experiment-esposalles', monitor=<Monitor.va_cer: 'va_cer'>, checkpoint=None), 'train': TrainArgs(delimiters=['<space>'], checkpoint_k=3, resume=False, early_stopping_patience=80, gpu_stats=False, augment_training=True)}
[2024-04-23 12:58:32,010 INFO laia] Installed:
[2024-04-23 12:58:32,050 INFO laia.common.loader] Loaded model model
[2024-04-23 12:58:32,094 INFO laia] Dataset is valid
[2024-04-23 12:58:32,094 INFO laia] Statistics written to statistics.md
-```
+----
-### Example with a YAML configuration file
+[#example-with-a-yaml-configuration-file]
+=== Example with a YAML configuration file
Run the following command to validate a dataset:
-```sh
+
+[,sh]
+----
pylaia-htr-dataset-validate --config config_dataset.yaml
-```
+----
Where `config_dataset.yaml` is:
-```yaml
+[,yaml]
+----
syms: /data/Esposalles/dataset/syms.txt
img_dirs: [/data/Esposalles/dataset/images/]
tr_txt_table: /data/Esposalles/dataset/train.txt
@@ -105,21 +197,25 @@ fixed_input_height: 128
statistics_output: statistics.md
common:
experiment_dirname: experiment-esposalles
-```
+----
-### Example with perfect dataset
+[#example-with-perfect-dataset]
+=== Example with perfect dataset
-```bash
+[,bash]
+----
[2024-04-23 12:58:31,399 INFO laia] Arguments: {'syms': '/data/Esposalles/dataset/syms.txt', 'img_dirs': ['/data/Esposalles/dataset/images/'], 'tr_txt_table': '/data/Esposalles/dataset/train.txt', 'va_txt_table': '/data/Esposalles/dataset/val.txt', 'te_txt_table': '/data/Esposalles/dataset/test.txt', 'fixed_input_height': 128, 'common': CommonArgs(seed=74565, train_path='', model_filename='model', experiment_dirname='experiment-esposalles', monitor=<Monitor.va_cer: 'va_cer'>, checkpoint=None), 'train': TrainArgs(delimiters=['<space>'], checkpoint_k=3, resume=False, early_stopping_patience=80, gpu_stats=False, augment_training=True)}
[2024-04-23 12:58:32,010 INFO laia] Installed:
[2024-04-23 12:58:32,050 INFO laia.common.loader] Loaded model model
[2024-04-23 12:58:32,094 INFO laia] Dataset is valid
[2024-04-23 12:58:32,094 INFO laia] Statistics written to statistics.md
-```
+----
-### Example with missing images
+[#example-with-missing-images]
+=== Example with missing images
-```bash
+[,bash]
+----
[2024-04-23 13:01:34,646 INFO laia] Arguments: {'syms': '/data/Esposalles/dataset/syms.txt', 'img_dirs': ['/data/Esposalles/dataset/images/'], 'tr_txt_table': '/data/Esposalles/dataset/train.txt', 'va_txt_table': '/data/Esposalles/dataset/val.txt', 'te_txt_table': '/data/Esposalles/dataset/test.txt', 'fixed_input_height': 128, 'common': CommonArgs(seed=74565, train_path='', model_filename='model', experiment_dirname='experiment-esposalles', monitor=<Monitor.va_cer: 'va_cer'>, checkpoint=None), 'train': TrainArgs(delimiters=['<space>'], checkpoint_k=3, resume=False, early_stopping_patience=80, gpu_stats=False, augment_training=True)}
[2024-04-23 13:01:35,200 INFO laia] Installed:
[2024-04-23 13:01:35,232 INFO laia.common.loader] Loaded model model
@@ -127,32 +223,38 @@ common:
[2024-04-23 13:01:35,783 WARNING laia.data.text_image_from_text_table_dataset] No image file found for image ID '0d7cf548-742b-4067-9084-52478806091d_Line0_b1fb9275-5d49-4266-9de0-e6a93fc6dfaf.jpg', ignoring example...
[2024-04-23 13:01:35,894 INFO laia] Dataset is valid
[2024-04-23 13:01:35,894 INFO laia] Statistics written to statistics.md
-```
+----
-### Example with small images
+[#example-with-small-images]
+=== Example with small images
-```sh
+[,sh]
+----
[2024-04-23 13:01:34,646 INFO laia] Arguments: {'syms': '/data/Esposalles/dataset/syms.txt', 'img_dirs': ['/data/Esposalles/dataset/images/'], 'tr_txt_table': '/data/Esposalles/dataset/train.txt', 'va_txt_table': '/data/Esposalles/dataset/val.txt', 'te_txt_table': '/data/Esposalles/dataset/test.txt', 'fixed_input_height': 128, 'common': CommonArgs(seed=74565, train_path='', model_filename='model', experiment_dirname='experiment-esposalles', monitor=<Monitor.va_cer: 'va_cer'>, checkpoint=None), 'train': TrainArgs(delimiters=['<space>'], checkpoint_k=3, resume=False, early_stopping_patience=80, gpu_stats=False, augment_training=True)}
[2024-04-23 13:01:35,200 INFO laia] Installed:
[2024-04-23 13:01:35,232 INFO laia.common.loader] Loaded model model
[2024-04-23 13:01:36,052 ERROR laia] Issues found in the dataset.
[2024-04-23 13:01:36,052 ERROR laia] train - Found some images too small for convolutions (width<8). They will be padded during training.
-```
+----
-### Example with variable image height
+[#example-with-variable-image-height]
+=== Example with variable image height
-```sh
+[,sh]
+----
[2024-04-23 13:01:34,646 INFO laia] Arguments: {'syms': '/data/Esposalles/dataset/syms.txt', 'img_dirs': ['/data/Esposalles/dataset/images/'], 'tr_txt_table': '/data/Esposalles/dataset/train.txt', 'va_txt_table': '/data/Esposalles/dataset/val.txt', 'te_txt_table': '/data/Esposalles/dataset/test.txt', 'fixed_input_height': 128, 'common': CommonArgs(seed=74565, train_path='', model_filename='model', experiment_dirname='experiment-esposalles', monitor=<Monitor.va_cer: 'va_cer'>, checkpoint=None), 'train': TrainArgs(delimiters=['<space>'], checkpoint_k=3, resume=False, early_stopping_patience=80, gpu_stats=False, augment_training=True)}
[2024-04-23 13:01:35,200 INFO laia] Installed:
[2024-04-23 13:01:35,232 INFO laia.common.loader] Loaded model model
[2024-04-23 13:01:36,052 ERROR laia] Issues found in the dataset.
[2024-04-23 13:01:36,052 ERROR laia] train - Found images with variable heights: ['/data/Esposalles/dataset/images/f6d2b699-e910-4191-bc7d-f56e60fe979a_Line2_91b43b71-ea60-4f42-a896-880676aed723.jpg'].
[2024-04-23 13:01:36,052 ERROR laia] test - Found images with variable heights: ['/data/Esposalles/dataset/images/fd1e6b3b-48cb-41c0-b1e9-2924b9562876_Line3_27e23ff1-f730-44ac-844f-479e5cc9e9aa.jpg'].
-```
+----
-### Example with missing symbol
+[#example-with-missing-symbol]
+=== Example with missing symbol
-```sh
+[,sh]
+----
[2024-04-23 13:01:34,646 INFO laia] Arguments: {'syms': '/data/Esposalles/dataset/syms.txt', 'img_dirs': ['/data/Esposalles/dataset/images/'], 'tr_txt_table': '/data/Esposalles/dataset/train.txt', 'va_txt_table': '/data/Esposalles/dataset/val.txt', 'te_txt_table': '/data/Esposalles/dataset/test.txt', 'fixed_input_height': 128, 'common': CommonArgs(seed=74565, train_path='', model_filename='model', experiment_dirname='experiment-esposalles', monitor=<Monitor.va_cer: 'va_cer'>, checkpoint=None), 'train': TrainArgs(delimiters=['<space>'], checkpoint_k=3, resume=False, early_stopping_patience=80, gpu_stats=False, augment_training=True)}
[2024-04-23 13:01:35,200 INFO laia] Installed:
[2024-04-23 13:01:35,232 INFO laia.common.loader] Loaded model model
@@ -160,4 +262,4 @@ common:
[2024-04-23 13:01:36,052 ERROR laia] train - Found some unknown symbols: {'='}
[2024-04-23 13:01:36,052 ERROR laia] val - Found some unknown symbols: {'='}
[2024-04-23 13:01:36,052 ERROR laia] test - Found some unknown symbols: {'='}
-```
+----
diff --git a/docs/modules/ROOT/pages/usage/index.adoc b/docs/modules/ROOT/pages/usage/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..ab241800a53c2eb4f2f73b87f26101bf3f48c2f1
--- /dev/null
+++ b/docs/modules/ROOT/pages/usage/index.adoc
@@ -0,0 +1,22 @@
+[#usage]
+= Usage
+
+Once the dataset is formatted and `pylaia` is installed and in your environment, you may use the following commands:
+
+* {blank}
+`pylaia-htr-create-model`:: To create a new PyLaia model. More details in the xref:./initialization/index.adoc[dedicated page].
+* {blank}
+`pylaia-htr-dataset-validate`:: To compute statistics and run validation checks on a dataset. More details in the xref:./datasets/index.adoc[dedicated page].
+* {blank}
+`pylaia-htr-train-ctc`:: To train a PyLaia model. More details in the xref:./training/index.adoc[dedicated page].
+* {blank}
+`pylaia-htr-decode-ctc`:: To predict using a trained PyLaia model. More details in the xref:./prediction/index.adoc[dedicated page].
+* {blank}
+`pylaia-htr-netout`:: To dump features from a PyLaia model. More details in the xref:./netout/index.adoc[dedicated page].
+
+'''
+
+Related pages:
+
+* Learn how to format a xref:./datasets/format.adoc[dataset in PyLaia format]
+* Learn how to use PyLaia with an xref:./language_models/index.adoc[explicit language model]
diff --git a/docs/modules/ROOT/pages/usage/initialization/index.adoc b/docs/modules/ROOT/pages/usage/initialization/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..60c5a3ac1af5f29ceb4c14477e343d4f7543cef4
--- /dev/null
+++ b/docs/modules/ROOT/pages/usage/initialization/index.adoc
@@ -0,0 +1,268 @@
+[#model-initialization]
+= Model initialization
+
+The `pylaia-htr-create-model` command can be used to create a PyLaia model. To know more about the options of this command, use `pylaia-htr-create-model --help`.
+
+[#purpose]
+== Purpose
+
+The general architecture of PyLaia is composed of convolutional blocks followed by a set a bi-directionnal recurrent layers and a linear layer. PyLaia is fully configurable by the user, including:
+
+* Number of convolutional blocks,
+* Number of recurrent layers,
+* Batch normalization,
+* Pooling layers,
+* Activation function,
+* ...
+
+This command will create a pickled file (named `model` by default), which is required to initialize the `LaiaCRNN` class before training.
+
+[#parameters]
+== Parameters
+
+The full list of parameters is detailed in this section.
+
+[#general-parameters]
+=== General parameters
+
+|===
+| Parameter | Description | Type | Default
+
+| `syms`
+| Positional argument. Path to a file mapping characters to integers. The CTC symbol must be mapped to integer 0.
+| `str`
+|
+
+| `config`
+| Path to a JSON configuration file
+| `json`
+|
+
+| `fixed_input_height`
+| Height of the input images. If set to 0, a variable height model will be used (see `adaptive_pooling`). This will be used to compute the model output height at the end of the convolutional layers.
+| `int`
+| 0
+
+| `adaptive_pooling`
+| Use custom adaptive pooling layers to enable training with variable height images. Takes into account the size of each individual image within the batch (before padding). Should be in `{avg,max}pool-N`.
+| `str`
+| `avgpool-16`
+
+| `save_model`
+| Whether to save the model to a file.
+| `bool`
+| `True`
+|===
+
+[#common-parameters]
+=== Common parameters
+
+|===
+| Name | Description | Type | Default
+
+| `common.train_path`
+| Directory where the model will be saved
+| `str`
+| `.`
+
+| `common.model_filename`
+| Filename of the model.
+| `str`
+| `model`
+|===
+
+[#logging-arguments]
+=== Logging arguments
+
+|===
+| Name | Description | Type | Default
+
+| `logging.fmt`
+| Logging format.
+| `str`
+| `%(asctime)s %(levelname)s %(name)s] %(message)s`
+
+| `logging.level`
+a| Logging level. Should be in
+
+* `NOTSET`
+* `DEBUG`
+* `INFO`
+* `WARNING`
+* `ERROR`
+* `CRITICAL`
+
+| `Level`
+| `INFO`
+
+| `logging.filepath`
+| Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname`
+| `Optional[str]`
+|
+
+| `logging.overwrite`
+| Whether to overwrite the logfile or to append.
+| `bool`
+| `False`
+
+| `logging.to_stderr_level`
+| If filename is set, use this to log also to stderr at the given level.
+| `Level`
+| `ERROR`
+|===
+
+[#architecture-arguments]
+=== Architecture arguments
+
+|===
+| Name | Description | Type | Default
+
+| `crnn.num_input_channels`
+| Number of channels of the input images.
+| `int`
+| `1`
+
+| `crnn.vertical_text`
+| Whether the text is written vertically.
+| `bool`
+| `False`
+
+| `crnn.cnn_num_features`
+| Number of features in each convolutional layer.
+| `List`
+| `[16, 16, 32, 32]`
+
+| `crnn.cnn_kernel_size`
+| Kernel size of each convolutional layer (e.g. [n,n,...] or [[h1,w1],[h2,w2],...]).
+| `List`
+| `[3, 3, 3, 3]`
+
+| `crnn.cnn_stride`
+| Stride of each convolutional layer. (e.g. [n,n,...] or [[h1,w1],[h2,w2],...])
+| `List`
+| `[1, 1, 1, 1]`
+
+| `crnn.cnn_dilation`
+| Spacing between each convolutional layer kernel elements. (e.g. [n,n,...] or [[h1,w1],[h2,w2],...])
+| `List`
+| `[1, 1, 1, 1]`
+
+| `crnn.cnn_activation`
+| Type of activation function in each convolutional layer (from `torch.nn`).
+| `List`
+| `['LeakyReLU', 'LeakyReLU', 'LeakyReLU', 'LeakyReLU']`
+
+| `crnn.cnn_poolsize`
+| MaxPooling size after each convolutional layer. (e.g. [n,n,...] or [[h1,w1],[h2,w2],...]).
+| `List`
+| `[2, 2, 2, 0]`
+
+| `crnn.cnn_dropout`
+| Dropout probability at the input of each convolutional layer.
+| `List`
+| `[0.0, 0.0, 0.0, 0.0]`
+
+| `crnn.cnn_batchnorm`
+| Whether to do batch normalization before the activation in each convolutional layer.
+| `List`
+| `[False, False, False, False]`
+
+| `crnn.rnn_layers`
+| Number of recurrent layers.
+| `int`
+| `3`
+
+| `crnn.rnn_units`
+| Number of units in each recurrent layer.
+| `int`
+| `256`
+
+| `crnn.rnn_dropout`
+| Dropout probability at the input of each recurrent layer.
+| `float`
+| `0.5`
+
+| `crnn.rnn_type`
+| Type of recurrent layer (from `torch.nn`).
+| `str`
+| `LSTM`
+
+| `crnn.lin_dropout`
+| Dropout probability at the input of the final linear layer.
+| `float`
+| `0.5`
+|===
+
+[#examples]
+== Examples
+
+The model can be configured using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
+
+[#example-with-command-line-arguments-cli]
+=== Example with Command Line Arguments (CLI)
+
+Run the following command to create a model:
+
+[,sh]
+----
+pylaia-htr-create-model /path/to/syms.txt \
+ --fixed_input_height 128 \
+ --crnn.rnn_layers 4 \
+ --logging.filepath model.log \
+ --common.train_path my_experiments/
+----
+
+[#example-with-a-yaml-configuration-file]
+=== Example with a YAML configuration file
+
+Run the following command to create a model:
+
+[,sh]
+----
+pylaia-htr-create-model --config config_create_model.yaml
+----
+
+Where `config_create_model.yaml` is:
+
+[,yaml]
+----
+crnn:
+ cnn_activation:
+ - LeakyReLU
+ - LeakyReLU
+ - LeakyReLU
+ - LeakyReLU
+ cnn_batchnorm:
+ - true
+ - true
+ - true
+ - true
+ cnn_dilation:
+ - 1
+ - 1
+ - 1
+ - 1
+ cnn_kernel_size:
+ - 3
+ - 3
+ - 3
+ - 3
+ cnn_num_features:
+ - 12
+ - 24
+ - 48
+ - 48
+ cnn_poolsize:
+ - 2
+ - 2
+ - 0
+ - 2
+ lin_dropout: 0.5
+ rnn_dropout: 0.5
+ rnn_layers: 3
+ rnn_type: LSTM
+ rnn_units: 256
+fixed_input_height: 128
+save_model: true
+syms: /path/to/syms.txt
+----
diff --git a/docs/usage/language_models/index.md b/docs/modules/ROOT/pages/usage/language_models/index.adoc
similarity index 78%
rename from docs/usage/language_models/index.md
rename to docs/modules/ROOT/pages/usage/language_models/index.adoc
index 11463ff18b39f856978d98b298d6b16ec258b09a..b1756bd1eba89c4d01e9ec2e41235d7365ed999b 100644
--- a/docs/usage/language_models/index.md
+++ b/docs/modules/ROOT/pages/usage/language_models/index.adoc
@@ -1,80 +1,96 @@
-# Explicit language modeling with n-grams
+[#explicit-language-modeling-with-n-grams]
+= Explicit language modeling with n-grams
PyLaia supports lattice rescoring using a statistical language model.
-This documentation gives instructions to build a language model with [kenlm](https://kheafield.com/code/kenlm/).
+This documentation gives instructions to build a language model with https://kheafield.com/code/kenlm/[kenlm].
-!!! note
- You can also use [SRILM](http://www.speech.sri.com/projects/srilm/) to build an ARPA language model.
+NOTE: You can also use http://www.speech.sri.com/projects/srilm/[SRILM] to build an ARPA language model.
To decode with a language model, you need:
-* [a language model](./index.md#build-the-language-model)
-* [a list of tokens](./index.md#list-of-tokens)
-* [a lexicon](./index.md#lexicon)
+* xref:./index.adoc#build-the-language-model[a language model]
+* xref:./index.adoc#list-of-tokens[a list of tokens]
+* xref:./index.adoc#lexicon[a lexicon]
-## Build the language model
+[#build-the-language-model]
+== Build the language model
-### Install kenlm
+[#install-kenlm]
+=== Install kenlm
-To build the language model, you first need to install and compile [kenlm](https://github.com/kpu/kenlm) by following the instructions detailed in the [README](https://github.com/kpu/kenlm#compiling).
+To build the language model, you first need to install and compile https://github.com/kpu/kenlm[kenlm] by following the instructions detailed in the https://github.com/kpu/kenlm#compiling[README].
-### Generate resources to train the language model
+[#generate-resources-to-train-the-language-model]
+=== Generate resources to train the language model
To train a language model, you need to generate a corpus containing the training text tokenized at character, subword or word level.
-#### Characters
+[#characters]
+==== Characters
Here is a sample of text tokenized at character-level (`corpus_characters.txt`).
-```text title="corpus_characters.txt"
+
+[,text]
+----
u d e <space> i <space> r e s t a u r a n t e r ,
v æ r e t <space> u h y r e <space> m e g e t <space> s a m m e n , <space> o f t e <space> t i l <space> m a a l t i d e r <space> o g <space> t i l <space> t h e <space> h o s <space> O s s b a h r ,
v i <space> s i d d e r <space> v e d <space> k a m i n e n <space> d e r <space> o g <space> s n a k k e r , <space> h v i l k e t <space> e r <space> m e g e t <space> m o r s o m t . <space> N u
k o m m e r <space> d e r <space> m a n g e <space> r e i s e n d e <space> v e n n e r <space> e l l e r <space> s l æ g t <space> e l l e r <space> p r i n s e s s e r , <space> s o m
O s s b a h r <space> m a a <space> v æ r e <space> s a m m e n <space> m e d <space> H e d b e r g <space> o f t e <space> o g s a a . <space> M e n <space> v i <space> k a n <space> l e v e
-```
+----
-#### Subwords
+[#subwords]
+==== Subwords
Here is a sample of text tokenized at subword-level (`corpus_subwords.txt`).
-```text title="corpus_subwords.txt"
+
+[,text]
+----
ud e <space> i <space> r e st au r ant er ,
været <space> u h y r e <space> meget <space> sammen , <space> ofte <space> til <space> ma altid er <space> og <space> til <space> th e <space> hos <space> O s s ba h r ,
vi <space> sidde r <space> ved <space> ka min en <space> der <space> og <space> snakke r , <space> hvilket <space> er <space> meget <space> morsomt . <space> Nu
kommer <space> der <space> mange <space> r e i sende <space> venner <space> eller <space> s læg t <space> eller <space> pr in s e s ser , <space> som
O s s ba h r <space> maa <space> være <space> sammen <space> med <space> H e d berg <space> ofte <space> ogsaa . <space> Men <space> vi <space> kan <space> lev e
-```
+----
+
+[#words]
+==== Words
-#### Words
Here is a sample of text tokenized at word-level (`corpus_words.txt`).
-```text title="corpus_words.txt"
+
+[,text]
+----
ude <space> i <space> restauranter <space> ,
været <space> uhyre <space> meget <space> sammen <space> , <space> ofte <space> til <space> maaltider <space> og <space> til <space> the <space> hos <space> Ossbahr <space> ,
vi <space> sidder <space> ved <space> kaminen <space> der <space> og <space> snakker <space> , <space> hvilket <space> er <space> meget <space> morsomt <space> . <space> Nu
kommer <space> der <space> mange <space> reisende <space> venner <space> eller <space> slægt <space> eller <space> prinsesser <space> , <space> som
Ossbahr <space> maa <space> være <space> sammen <space> med <space> Hedberg <space> ofte <space> ogsaa <space> . <space> Men <space> vi <space> kan <space> leve
-```
+----
-### Train the language model
+[#train-the-language-model]
+=== Train the language model
Once your corpus is created, you can estimate the n-gram model.
-#### Characters
+[#training-characters]
+==== Characters
At character-level, we recommend building a 6-gram model. Use the following command:
-```sh
+[,sh]
+----
bin/lmplz --order 6 \
--text my_dataset/language_model/corpus_characters.txt \
--arpa my_dataset/language_model/model_characters.arpa \
--discount_fallback
-```
+----
-!!! note
- The `--discount_fallback` option can be removed if your corpus is very large.
+NOTE: The `--discount_fallback` option can be removed if your corpus is very large.
The following message should be displayed if the language model was built successfully:
-```sh
+[,sh]
+----
=== 1/5 Counting and sorting n-grams ===
Reading language_model/corpus.txt
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
@@ -109,52 +125,58 @@ Chain sizes: 1:1308 2:27744 3:159140 4:412536 5:717920 6:1028896
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
****************************************************************************************************
Name:lmplz VmPeak:12643224 kB VmRSS:6344 kB RSSMax:1969316 kB user:0.196445 sys:0.514686 CPU:0.711161 real:0.682693
-```
+----
-#### Subwords
+[#training-subwords]
+==== Subwords
At subword-level, we recommend building a 6-gram model. Use the following command:
-```sh
+[,sh]
+----
bin/lmplz --order 6 \
--text my_dataset/language_model/corpus_subwords.txt \
--arpa my_dataset/language_model/model_subwords.arpa \
--discount_fallback
-```
+----
-!!! note
- The `--discount_fallback` option can be removed if your corpus is very large.
+NOTE: The `--discount_fallback` option can be removed if your corpus is very large.
-#### Words
+[#words-2]
+==== Words
At word-level, we recommend building a 3-gram model. Use the following command:
-```sh
+[,sh]
+----
bin/lmplz --order 3 \
--text my_dataset/language_model/corpus_words.txt \
--arpa my_dataset/language_model/model_words.arpa \
--discount_fallback
-```
+----
-!!! note
- The `--discount_fallback` option can be removed if your corpus is very large.
+NOTE: The `--discount_fallback` option can be removed if your corpus is very large.
-## Predict with a language model
+[#predict-with-a-language-model]
+== Predict with a language model
Once the language model is trained, you need to generate a list of tokens and a lexicon.
-### List of tokens
+[#list-of-tokens]
+=== List of tokens
The list of tokens `tokens.txt` lists all the tokens that can be predicted by PyLaia.
It should be similar to `syms.txt`, but without any index, and can be generated with this command:
-```bash
+
+[,bash]
+----
cut -d' ' -f 1 syms.txt > tokens.txt
-```
+----
-!!! note
- This file does not depend on the tokenization level.
+NOTE: This file does not depend on the tokenization level.
-```text title="tokens.txt"
+[,text]
+----
<ctc>
.
,
@@ -163,17 +185,20 @@ b
c
...
<space>
-```
+----
-### Lexicon
+[#lexicon]
+=== Lexicon
The lexicon lists all the words in the vocabulary and its decomposition in tokens.
-#### Characters
+[#prediction-characters-2]
+==== Characters
At character-level, words are simply characters, so the `lexicon_characters.txt` file should map characters to characters:
-```text title="lexicon_characters.txt"
+[,text]
+----
<ctc> <ctc>
. .
, ,
@@ -182,12 +207,15 @@ b b
c c
...
<space> <space>
-```
+----
+
+[#prediction-subwords]
+==== Subwords
-#### Subwords
At subword-level, the `lexicon_subwords.txt` file should map subwords with their character decomposition:
-```text title="lexicon_subwords.txt"
+[,text]
+----
<ctc> <ctc>
. .
, ,
@@ -196,12 +224,15 @@ ant a n t
au a u
...
<space> <space>
-```
+----
+
+[#prediction-words]
+==== Words
-#### Words
At word-level, the `lexicon_words.txt` file should map words with their character decomposition:
-```text title="lexicon_words.txt"
+[,text]
+----
<ctc> <ctc>
. .
, ,
@@ -210,8 +241,9 @@ er e r
eller e l l e r
...
<space> <space>
-```
+----
-### Predict with PyLaia
+[#predict-with-pylaia]
+=== Predict with PyLaia
-See the [dedicated example](../prediction/index.md#predict-with-a-language-model).
+See the xref:usage/prediction/index.adoc#predict-with-a-language-model[dedicated example].
diff --git a/docs/modules/ROOT/pages/usage/netout/index.adoc b/docs/modules/ROOT/pages/usage/netout/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..fe7d857cb5c91ac7936334418ccabb31a7489895
--- /dev/null
+++ b/docs/modules/ROOT/pages/usage/netout/index.adoc
@@ -0,0 +1,244 @@
+[#netout]
+= Netout
+
+The `pylaia-htr-netout` command can be used to dump the features extracted by PyLaia for a set of text-lines. To know more about the options of this command, use `pylaia-htr-netout --help`.
+
+WARNING: This command was initially designed to combine PyLaia and Kaldi. Since December 2022, combining PyLaia with language models can be achieved more easily by xref:usage/language_models/index.adoc[building a language model with KenLM] and xref:usage/prediction/index.adoc#predict-with-a-language-model[predicting with `pylaia-htr-decode-ctc`].
+
+[#purpose]
+== Purpose
+
+This command outputs the feature matrix and lattice computed by PyLaia in Kaldi format for a given dataset.
+
+It requires:
+
+* a xref:usage/datasets/index.adoc#image-names[list of image ids],
+* the pickled `model` file created during xref:usage/initialization/index.adoc[model initialization],
+* the weights `*.ckpt` of the trained model created during xref:usage/training/index.adoc[model training].
+
+The files generated by this command are designed to combine PyLaia and Kaldi, but could also be used to predict with a custom decoder.
+
+[#parameters]
+== Parameters
+
+The full list of parameters is detailed in this section.
+
+[#general-parameters]
+=== General parameters
+
+|===
+| Parameter | Description | Type | Default
+
+| `img_list`
+| Positional argument. File containing the names of the images to decode (one image per line).
+| `str`
+|
+
+| `img_dirs`
+| Directories containing line images.
+| `str`
+|
+
+| `config`
+| Path to a JSON configuration file
+| `json`
+|
+|===
+
+[#common-parameters]
+=== Common parameters
+
+|===
+| Name | Description | Type | Default
+
+| `common.train_path`
+| Directory where the model will be saved
+| `str`
+| `.`
+
+| `common.model_filename`
+| Filename of the model.
+| `str`
+| `model`
+
+| `common.experiment_dirname`
+| Directory name of the experiment.
+| `experiment`
+| `74565`
+
+| `common.checkpoint`
+| Checkpoint to load. Must be a filepath, a filename, a glob pattern or `None` (in this case, the best checkpoint will be loaded). Note that the checkpoint will be searched in `common.experiment_dirname`, unless you provide an absolute filepath.
+| `int`
+| `None`
+|===
+
+[#data-arguments]
+=== Data arguments
+
+|===
+| Name | Description | Type | Default
+
+| `data.batch_size`
+| Batch size.
+| `int`
+| `8`
+
+| `data.color_mode`
+| Color mode. Must be either `L`, `RGB` or `RGBA`.
+| `ColorMode`
+| `ColorMode.L`
+|===
+
+[#netout-arguments]
+=== Netout arguments
+
+|===
+| Name | Description | Type | Default
+
+| `netout.output_transform`
+| Transformation to apply at the end of the model. Should be `softmax` or `log_softmax`.
+| `str`
+| `None`
+
+| `netout.matrix`
+| Path to the output file containing a list of keys (image ids) and values (output matrix where rows represents timesteps and columns CTC labels). This file can be directly used with Kaldi.
+| `Optional[str]`
+| `None`
+
+| `netout.lattice`
+| Path to the output file containing containing a list of keys (image ids) and values (lattices representing the CTC output). This file can be directly used with Kaldi.
+| `Optional[str]`
+| `None`
+
+| `netout.digits`
+| Number of digits to be used for formatting
+| `int`
+| `10`
+|===
+
+[#logging-arguments]
+=== Logging arguments
+
+|===
+| Name | Description | Type | Default
+
+| `logging.fmt`
+| Logging format.
+| `str`
+| `%(asctime)s %(levelname)s %(name)s] %(message)s`
+
+| `logging.level`
+a| Logging level. Should be in
+
+* `NOTSET`
+* `DEBUG`
+* `INFO`
+* `WARNING`
+* `ERROR`
+* `CRITICAL`
+
+| `Level`
+| `INFO`
+
+| `logging.filepath`
+| Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname`
+| `Optional[str]`
+|
+
+| `logging.overwrite`
+| Whether to overwrite the logfile or to append.
+| `bool`
+| `False`
+
+| `logging.to_stderr_level`
+| If filename is set, use this to log also to stderr at the given level.
+| `Level`
+| `ERROR`
+|===
+
+[#trainer-arguments]
+=== Trainer arguments
+
+Pytorch Lightning `Trainer` flags can also be set using the `--trainer` argument. See https://github.com/Lightning-AI/lightning/blob/1.7.0/docs/source-pytorch/common/trainer.rst#trainer-flags[the documentation].
+
+This flag is mostly useful to define whether to predict on CPU or GPU.
+
+* `--trainer.gpus 0` to run on CPU,
+* `--trainer.gpus n` to run on `n` GPUs (use with `--training.auto_select True` for auto-selection),
+* `--trainer.gpus -1` to run on all GPUs.
+
+[#examples]
+== Examples
+
+Dumping PyLaia's features can be done using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
+
+[#dumping-features-from-a-model-from-hugging-face]
+=== Dumping features from a model from Hugging Face
+
+First, clone a trained model from Hugging Face:
+
+[,bash]
+----
+git clone https://huggingface.co/Teklia/pylaia-huginmunin
+----
+
+List image names in `img_list.txt`:
+
+[,text]
+----
+docs/assets/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f
+docs/assets/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4
+----
+
+Dump PyLaia's output with:
+
+[,bash]
+----
+pylaia-htr-netout --common.experiment_dirname pylaia-huginmunin/ \
+ --common.model_filename pylaia-huginmunin/model \
+ --netout.matrix matrix.txt \
+ --netout.lattice lattice.txt \
+ --img_dir [docs/assets] \
+ img_list.txt
+----
+
+Output files will be written in `--common.experiment_dirname`:
+
+----
+├── pylaia-huginmunin/
+ ├── matrix.txt
+ └── lattice.txt
+----
+
+[#dumping-features-using-a-yaml-configuration-file]
+=== Dumping features using a YAML configuration file
+
+Run the following command to dump PyLaia's output:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --config config_netout.yaml
+----
+
+With the following configuration file:
+
+[,yaml]
+----
+common:
+ experiment_dirname: pylaia-huginmunin
+ model_filename: pylaia-huginmunin/model
+img_list: img_list.txt
+img_dirs:
+ - docs/assets/
+netout:
+ matrix: matrix.txt
+ lattice: lattice.txt
+----
+
+Output files will be written in `--common.experiment_dirname`:
+
+----
+├── pylaia-huginmunin/
+ ├── matrix.txt
+ └── lattice.txt
+----
diff --git a/docs/modules/ROOT/pages/usage/prediction/index.adoc b/docs/modules/ROOT/pages/usage/prediction/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..ee4ccd09fe087e7148337e397ef425bf15bdf5e2
--- /dev/null
+++ b/docs/modules/ROOT/pages/usage/prediction/index.adoc
@@ -0,0 +1,538 @@
+[#decoding]
+= Decoding
+
+The `pylaia-htr-decode-ctc` command can be used to predict using a trained PyLaia model. To know more about the options of this command, use `pylaia-htr-decode-ctc --help`.
+
+[#purpose]
+== Purpose
+
+This command uses a trained PyLaia model to predict on a dataset.
+
+It requires:
+
+* a xref:usage/datasets/index.adoc#image-names[list of image ids],
+* the pickled `model` file created during xref:usage/initialization/index.adoc[model initialization],
+* the weights `*.ckpt` of the trained model created during xref:usage/training/index.adoc[model training].
+
+[#parameters]
+== Parameters
+
+The full list of parameters is detailed in this section.
+
+[#general-parameters]
+=== General parameters
+
+|===
+| Parameter | Description | Type | Default
+
+| `syms`
+| Positional argument. Path to a file mapping characters to integers. The CTC symbol *must* be mapped to integer 0.
+| `str`
+|
+
+| `img_list`
+| Positional argument. File containing the names of the images to decode (one image per line).
+| `str`
+|
+
+| `img_dirs`
+| Directories containing line images.
+| `str`
+|
+
+| `config`
+| Path to a JSON configuration file
+| `json`
+|
+|===
+
+[#common-parameters]
+=== Common parameters
+
+|===
+| Name | Description | Type | Default
+
+| `common.train_path`
+| Directory where the model will be saved
+| `str`
+| `.`
+
+| `common.model_filename`
+| Filename of the model.
+| `str`
+| `model`
+
+| `common.experiment_dirname`
+| Directory name of the experiment.
+| `experiment`
+| `74565`
+
+| `common.checkpoint`
+| Checkpoint to load. Must be a filepath, a filename, a glob pattern or `None` (in this case, the best checkpoint will be loaded). Note that the checkpoint will be searched in `common.experiment_dirname`, unless you provide an absolute filepath.
+| `int`
+| `None`
+|===
+
+[#data-arguments]
+=== Data arguments
+
+|===
+| Name | Description | Type | Default
+
+| `data.batch_size`
+| Batch size.
+| `int`
+| `8`
+
+| `data.color_mode`
+| Color mode. Must be either `L`, `RGB` or `RGBA`.
+| `ColorMode`
+| `ColorMode.L`
+
+| `data.num_workers`
+| Number of worker processes created in dataloaders
+| `int`
+| `None`
+
+| `data.reading_order`
+| Reading order on the input lines: LTR (Left-to-Right) or RTL (Right-to-Left).
+| `ReadingOrder`
+| `LTR`
+|===
+
+[#decode-arguments]
+=== Decode arguments
+
+|===
+| Name | Description | Type | Default
+
+| `decode.include_img_ids`
+| Include the associated image ids in the decoding/segmentation output
+| `bool`
+| `True`
+
+| `decode.separator`
+| String to use as a separator between the image ids and the decoding/segmentation output.
+| `str`
+| ` `
+
+| `decode.join_string`
+| String to use to join the decoding output.
+| `Optional[str]`
+| ` `
+
+| `decode.use_symbols`
+| Convert the decoding output to symbols instead of symbol index.
+| `bool`
+| `True`
+
+| `decode.convert_spaces`
+| Whether or not to convert spaces.
+| `bool`
+| `False`
+
+| `decode.input_space`
+| Replace the space by this symbol if `convert_spaces` is set. Used for word segmentation and confidence score computation.
+| `str`
+| `<space>`
+
+| `decode.output_space`
+| Space symbol to display during decoding.
+| `str`
+| ` `
+
+| `decode.segmentation`
+| Use CTC alignment to estimate character or word segmentation. Should be `char` or `word`.
+| `Optional[str]`
+| `None `
+
+| `decode.temperature`
+| Temperature parameters used to scale the logits.
+| `float`
+| `1.0`
+
+| `decode.print_line_confidence_scores`
+| Whether to print line confidence scores.
+| `bool`
+| `False`
+
+| `decode.print_line_confidence_scores`
+| Whether to print word confidence scores.
+| `bool`
+| `False`
+
+| `decode.use_language_model`
+| Whether to decode with an external language model.
+| `bool`
+| `False`
+
+| `decode.language_model_path`
+| Path to a KenLM or ARPA n-gram language model.
+| `str`
+| `None`
+
+| `decode.language_model_weight`
+| Weight of the language model.
+| `float`
+| `None`
+
+| `decode.tokens_path`
+| Path to a file containing valid tokens. If using a file, the expected format is for tokens mapping to the same index to be on the same line. The `ctc` symbol should be at index 0.
+| `str`
+| `None`
+
+| `decode.lexicon_path`
+| Path to a lexicon file containing the possible words and corresponding spellings.
+| `str`
+| `None`
+
+| `decode.unk_token`
+| String representing unknown characters.
+| `str`
+| `<unk>`
+
+| `decode.blank_token`
+| String representing the blank/ctc symbol.
+| `str`
+| `<ctc>`
+|===
+
+[#logging-arguments]
+=== Logging arguments
+
+|===
+| Name | Description | Type | Default
+
+| `logging.fmt`
+| Logging format.
+| `str`
+| `%(asctime)s %(levelname)s %(name)s] %(message)s`
+
+| `logging.level`
+a| Logging level. Should be in
+
+* `NOTSET`
+* `DEBUG`
+* `INFO`
+* `WARNING`
+* `ERROR`
+* `CRITICAL`
+
+| `Level`
+| `INFO`
+
+| `logging.filepath`
+| Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname`
+| `Optional[str]`
+|
+
+| `logging.overwrite`
+| Whether to overwrite the logfile or to append.
+| `bool`
+| `False`
+
+| `logging.to_stderr_level`
+| If filename is set, use this to log also to stderr at the given level.
+| `Level`
+| `ERROR`
+|===
+
+[#trainer-arguments]
+=== Trainer arguments
+
+Pytorch Lightning `Trainer` flags can also be set using the `--trainer` argument. See https://github.com/Lightning-AI/lightning/blob/1.7.0/docs/source-pytorch/common/trainer.rst#trainer-flags[the documentation].
+
+This flag is mostly useful to define whether to predict on CPU or GPU.
+
+* `--trainer.gpus 0` to run on CPU,
+* `--trainer.gpus n` to run on `n` GPUs (use with `--training.auto_select True` for auto-selection),
+* `--trainer.gpus -1` to run on all GPUs.
+
+[#examples]
+== Examples
+
+The prediction can be done using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
+
+We provide some images to try out our models. They can be found in `docs/assets`, on the https://gitlab.teklia.com/atr/pylaia/-/tree/master/docs/assets?ref_type=heads[Gitlab repository]. To test the prediction commands, make sure to download them on your end.
+
+[,shell]
+----
+mkdir images
+wget https://user-images.githubusercontent.com/100838858/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f.jpg -P images
+wget https://user-images.githubusercontent.com/100838858/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4.jpg -P images
+----
+
+[#predict-using-a-model-from-hugging-face]
+=== Predict using a model from Hugging Face
+
+First, clone a trained model from Hugging Face:
+
+[,bash]
+----
+git clone https://huggingface.co/Teklia/pylaia-huginmunin
+----
+
+[NOTE]
+====
+Some files are stored through https://git-lfs.com/[Git-LFS]. Make sure all files are correctly pulled using the following command, from the cloned folder.
+
+[,bash]
+----
+git lfs ls-files
+----
+
+You should see three files:
+
+* the language model (`language_model.arpa.gz`),
+* the model architecture (`model`),
+* the weights (`weights.ckpt`).
+====
+
+List image names in `img_list.txt`:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4
+----
+
+Predict with:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --common.experiment_dirname pylaia-huginmunin/ \
+ --common.model_filename pylaia-huginmunin/model \
+ --img_dir [images] \
+ pylaia-huginmunin/syms.txt \
+ img_list.txt
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f o g <space> V a l s t a d <space> k a n <space> v i <space> v i s t
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 i k k e <space> g j ø r e <space> R e g n i n g <space> p a a ,
+----
+
+Note that by default, each token is separated by a space, and the space symbol is represented by `--decode.input_space` (default: `"<space>"`).
+
+[#predict-with-a-yaml-configuration-file]
+=== Predict with a YAML configuration file
+
+Run the following command to predict a model on CPU using:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --config config_decode_model.yaml
+----
+
+With the following configuration file:
+
+[,yaml]
+----
+syms: pylaia-huginmunin/syms.txt
+img_list: img_list.txt
+img_dirs:
+ - images/
+common:
+ experiment_dirname: pylaia-huginmunin
+ model_filename: pylaia-huginmunin/model
+decode:
+ join_string: ""
+ convert_spaces: true
+trainer:
+ gpus: 0
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f og Valstad kan vi vist
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 ikke gjøre Regning paa,
+----
+
+Note that setting `--decode.join_string ""` and `--decode.convert_spaces True` will display the text well formatted.
+
+[#predict-with-confidence-scores]
+=== Predict with confidence scores
+
+PyLaia estimate character probability for each timestep. It is possible to print the probability at line or word level.
+
+[#line-confidence-scores]
+==== Line confidence scores
+
+Run the following command to predict with line confidence scores:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --config config_decode_model.yaml \
+ --decode.print_line_confidence_score True
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f 0.99 og Valstad kan vi vist
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 0.98 ikke gjøre Regning paa,
+----
+
+[#word-confidence-scores]
+==== Word confidence scores
+
+Run the following command to predict with word confidence scores:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --config config_decode_model.yaml \
+ --decode.print_word_confidence_score True
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f ['1.00', '1.00', '1.00', '1.00', '1.00'] og Valstad kan vi vist
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 ['1.00', '0.91', '1.00', '0.99'] ikke gjøre Regning paa,
+----
+
+[#temperature-scaling]
+==== Temperature scaling
+
+PyLaia tends to output overly confident probabilities. https://arxiv.org/pdf/1706.04599.pdf[Temperature scaling] can be used to improve the reliability of confidence scores. The best temperature can be determined with a grid search algorithm by maximizing the correlation between 1-CER and confidence scores.
+
+Run the following command to predict callibrated word confidence scores with `temperature=3.0`
+
+[,bash]
+----
+pylaia-htr-decode-ctc --config config_decode_model.yaml \
+ --decode.print_word_confidence_score True \
+ --decode.temperature 3.0
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f ['0.93', '0.85', '0.87', '0.93', '0.85'] og Valstad kan vi vist
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 ['0.93', '0.84', '0.86', '0.83'] ikke gjøre Regning paa,
+----
+
+[#predict-with-a-language-model]
+=== Predict with a language model
+
+PyLaia supports KenLM and ARPA language models.
+
+Once the n-gram model is built, run the following command to combine it to your PyLaia model:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --config config_decode_model_lm.yaml
+----
+
+With the following configuration file:
+
+[,yaml]
+----
+syms: pylaia-huginmunin/syms.txt
+img_list: img_list.txt
+img_dirs:
+ - images/
+common:
+ experiment_dirname: pylaia-huginmunin
+ model_filename: pylaia-huginmunin/model
+decode:
+ join_string: ""
+ convert_spaces: true
+ use_language_model: true
+ language_model_path: pylaia-huginmunin/language_model.arpa.gz
+ tokens_path: pylaia-huginmunin/tokens.txt
+ lexicon_path: pylaia-huginmunin/lexicon.txt
+ language_model_weight: 1.5
+ decode.print_line_confidence_score: true
+trainer:
+ gpus: 0
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f 0.90 og Valstad kan vi vist
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 0.89 ikke gjøre Regning paa,
+----
+
+[#predict-with-ctc-alignement]
+=== Predict with CTC alignement
+
+It is possible to estimate text localization based on CTC alignments with the `--decode.segmentation` option. It returns a list texts with their estimated coordinates: `(text, x1, y1, x2, y2)`.
+
+[#character-level]
+==== Character level
+
+To output character localization, use the `--decode.segmentation char` option:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --common.experiment_dirname pylaia-huginmunin/ \
+ --common.model_filename pylaia-huginmunin/model \
+ --decode.segmentation char \
+ --img_dir [images] \
+ pylaia-huginmunin/syms.txt \
+ img_list.txt
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f [('o', 1, 1, 31, 128), ('g', 32, 1, 79, 128), ('<space>', 80, 1, 143, 128), ('V', 144, 1, 167, 128), ('a', 168, 1, 223, 128), ('l', 224, 1, 255, 128), ('s', 256, 1, 279, 128), ('t', 280, 1, 327, 128), ('a', 328, 1, 367, 128), ('d', 368, 1, 407, 128), ('<space>', 408, 1, 496, 128), ('k', 497, 1, 512, 128), ('a', 513, 1, 576, 128), ('n', 577, 1, 624, 128), ('<space>', 625, 1, 712, 128), ('v', 713, 1, 728, 128), ('i', 729, 1, 776, 128), ('<space>', 777, 1, 808, 128), ('v', 809, 1, 824, 128), ('i', 825, 1, 872, 128), ('s', 873, 1, 912, 128), ('t', 913, 1, 944, 128)]
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 [('i', 1, 1, 23, 128), ('k', 24, 1, 71, 128), ('k', 72, 1, 135, 128), ('e', 136, 1, 191, 128), ('<space>', 192, 1, 248, 128), ('g', 249, 1, 264, 128), ('j', 265, 1, 312, 128), ('ø', 313, 1, 336, 128), ('r', 337, 1, 376, 128), ('e', 377, 1, 408, 128), ('<space>', 409, 1, 481, 128), ('R', 482, 1, 497, 128), ('e', 498, 1, 545, 128), ('g', 546, 1, 569, 128), ('n', 570, 1, 601, 128), ('i', 602, 1, 665, 128), ('n', 666, 1, 706, 128), ('g', 707, 1, 762, 128), ('<space>', 763, 1, 794, 128), ('p', 795, 1, 802, 128), ('a', 803, 1, 850, 128), ('a', 851, 1, 890, 128), (',', 891, 1, 914, 128)]
+----
+
+[#word-level]
+==== Word level
+
+To output word localization, use the `--decode.segmentation word` option:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --common.experiment_dirname pylaia-huginmunin/ \
+ --common.model_filename pylaia-huginmunin/model \
+ --decode.segmentation word \
+ --img_dir [images] \
+ pylaia-huginmunin/syms.txt \
+ img_list.txt
+----
+
+Expected output:
+
+[,text]
+----
+219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f [('og', 1, 1, 79, 128), ('<space>', 80, 1, 143, 128), ('Valstad', 144, 1, 407, 128), ('<space>', 408, 1, 496, 128), ('kan', 497, 1, 624, 128), ('<space>', 625, 1, 712, 128), ('vi', 713, 1, 776, 128), ('<space>', 777, 1, 808, 128), ('vist', 809, 1, 944, 128)]
+219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 [('ikke', 1, 1, 191, 128), ('<space>', 192, 1, 248, 128), ('gjøre', 249, 1, 408, 128), ('<space>', 409, 1, 481, 128), ('Regning', 482, 1, 762, 128), ('<space>', 763, 1, 794, 128), ('paa,', 795, 1, 914, 128)]
+----
+
+[#predict-on-right-to-left-data]
+=== Predict on Right-To-Left data
+
+To output word localization, use the `--data.reading_order` option:
+
+[,bash]
+----
+pylaia-htr-decode-ctc --common.experiment_dirname pylaia-khatt/ \
+ --common.model_filename pylaia-khatt/model \
+ --data.reading_order RTL \
+ --img_dir [images] \
+ pylaia-khatt/syms.txt \
+ img_list.txt
+----
+
+Expected output:
+
+[,text]
+----
+text_line_1302 العلماء على Ùهم هذه الكتابات بالدراسات اللغوية السامية مثل العبرانية، وباللغة العربية التي
+----
diff --git a/docs/modules/ROOT/pages/usage/training/index.adoc b/docs/modules/ROOT/pages/usage/training/index.adoc
new file mode 100644
index 0000000000000000000000000000000000000000..3693cd3ab1780b7c4bcf437feadbc3ed1639afc6
--- /dev/null
+++ b/docs/modules/ROOT/pages/usage/training/index.adoc
@@ -0,0 +1,419 @@
+[#training]
+= Training
+
+The `pylaia-htr-train-ctc` command can be used to train a PyLaia model. To know more about the options of this command, use `pylaia-htr-train-ctc --help`.
+
+[#purpose]
+== Purpose
+
+This command trains a PyLaia architecture on a dataset.
+
+It requires:
+
+* a xref:usage/datasets/index.adoc[formatted dataset],
+* the pickled `model` file created during xref:usage/initialization/index.adoc[model initialization].
+
+NOTE: The xref:usage/datasets/index.adoc[`pylaia-htr-dataset-validate`] command can help you analyze your dataset and point out issues.
+
+[#parameters]
+== Parameters
+
+The full list of parameters is detailed in this section.
+
+[#general-parameters]
+=== General parameters
+
+|===
+| Parameter | Description | Type | Default
+
+| `syms`
+| Positional argument. Path to a file mapping characters to integers. The CTC symbol *must* be mapped to integer 0.
+| `str`
+|
+
+| `img_dirs`
+| Positional argument. Directories containing line images.
+| `str`
+|
+
+| `tr_txt_table`
+| Positional argument. Path to a file mapping training image ids and tokenized transcription.
+| `str`
+|
+
+| `va_txt_table`
+| Positional argument. Path to a file mapping validation image ids and tokenized transcription.
+| `str`
+|
+
+| `config`
+| Path to a JSON configuration file
+| `json`
+|
+|===
+
+[#common-parameters]
+=== Common parameters
+
+|===
+| Name | Description | Type | Default
+
+| `common.seed`
+| Seed for random number generators.
+| `int`
+| `74565`
+
+| `common.train_path`
+| Directory where the model will be saved
+| `str`
+| `.`
+
+| `common.model_filename`
+| Filename of the model.
+| `str`
+| `model`
+
+| `common.experiment_dirname`
+| Directory name of the experiment.
+| `str`
+| `experiment`
+
+| `common.monitor`
+| Metric to monitor for early stopping and checkpointing.
+| `Monitor`
+| `Monitor.va_cer`
+
+| `common.checkpoint`
+| Checkpoint to load. Must be a filepath, a filename, a glob pattern or `None` (in this case, the best checkpoint will be loaded). Note that the checkpoint will be searched in `common.experiment_dirname`, unless you provide an absolute filepath.
+| `Optional[str]`
+| `None`
+|===
+
+[#data-arguments]
+=== Data arguments
+
+|===
+| Name | Description | Type | Default
+
+| `data.batch_size`
+| Batch size.
+| `int`
+| `8`
+
+| `data.color_mode`
+| Color mode. Must be either `L`, `RGB` or `RGBA`.
+| `ColorMode`
+| `ColorMode.L`
+
+| `data.num_workers`
+| Number of worker processes created in dataloaders
+| `int`
+| `None`
+
+| `data.reading_order`
+| Reading order on the input lines: LFT (Left-to-Right) or RTL (Right-to-Left).
+| `ReadingOrder`
+| `LFT`
+|===
+
+[#train-arguments]
+=== Train arguments
+
+|===
+| Name | Description | Type | Default
+
+| `train.delimiters`
+| List of symbols representing the word delimiters.
+| `List`
+| `["<space>"]`
+
+| `train.checkpoint_k`
+| Model saving mode: `-1` all models will be saved, `0`: no models are saved, `k` the `k` best models are saved.
+| `int`
+| `3`
+
+| `train.resume`
+| Whether to resume training with a checkpoint. This option can be used to continue training on the same dataset.
+| `bool`
+| `False`
+
+| `train.pretrain`
+| Whether to load pretrained weights from a checkpoint. This option can be used to load pretrained weights when fine-tuning a model on a new dataset.
+| `bool`
+| `False`
+
+| `train.freeze_layers`
+| List of layers to freeze during training: `"conv"` to freeze convolutional layers, `"rnn"` to freeze recurrent layers, `"linear"` to freeze the linear layer
+| `List[str]`
+| `None`
+
+| `train.early_stopping_patience`
+| Number of validation epochs with no improvement after which training will be stopped.
+| `int`
+| `20`
+
+| `train.gpu_stats`
+| Whether to include GPU stats in the training progress bar.
+| `bool`
+| `False`
+
+| `train.augment_training`
+| Whether to use data augmentation.
+| `bool`
+| `False`
+
+| `train.log_to_wandb`
+| Whether to log training metrics and parameters to Weights & Biases.
+| `bool`
+| `False`
+|===
+
+[#logging-arguments]
+=== Logging arguments
+
+|===
+| Name | Description | Type | Default
+
+| `logging.fmt`
+| Logging format.
+| `str`
+| `%(asctime)s %(levelname)s %(name)s] %(message)s`
+
+| `logging.level`
+a| Logging level. Should be in
+
+* `NOTSET`
+* `DEBUG`
+* `INFO`
+* `WARNING`
+* `ERROR`
+* `CRITICAL`
+
+| `Level`
+| `INFO`
+
+| `logging.filepath`
+| Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname`
+| `Optional[str]`
+|
+
+| `logging.overwrite`
+| Whether to overwrite the logfile or to append.
+| `bool`
+| `False`
+
+| `logging.to_stderr_level`
+| If filename is set, use this to log also to stderr at the given level.
+| `Level`
+| `ERROR`
+|===
+
+[#optimizer-arguments]
+=== Optimizer arguments
+
+|===
+| Name | Description | Type | Default
+
+| `optimizers.name`
+| Optimization algorithm. Must be `SGD`, `RMSProp`, `Adam`.
+| `List`
+| `RMSProp`
+
+| `optimizers.learning_rate`
+| Learning rate.
+| `float`
+| `0.0005`
+
+| `optimizers.momentum`
+| Momentum.
+| `float`
+| `0.0`
+
+| `optimizers.weight_l2_penalty`
+| Apply this L2 weight penalty to the loss function.
+| `float`
+| `0.0`
+
+| `optimizers.nesterov`
+| Whether to use Nesterov momentum.
+| `bool`
+| `False`
+|===
+
+[#scheduler-arguments]
+=== Scheduler arguments
+
+|===
+| Name | Description | Type | Default
+
+| `scheduler.active`
+| Whether to use an on-plateau learning rate scheduler.
+| `bool`
+| `False`
+
+| `scheduler.monitor`
+| Metric for the scheduler to monitor.
+| `Monitor`
+| `Monitor.va_loss`
+
+| `scheduler.patience`
+| Number of epochs with no improvement after which learning rate will be reduced.
+| `int`
+| `5`
+
+| `scheduler.factor`
+| Factor by which the learning rate will be reduced.
+| `float`
+| `0.1`
+|===
+
+[#trainer-arguments]
+=== Trainer arguments
+
+Pytorch Lighning `Trainer` flags can also be set using the `--trainer` argument. See https://github.com/Lightning-AI/lightning/blob/1.7.0/docs/source-pytorch/common/trainer.rst#trainer-flags[the documentation].
+
+[#examples]
+== Examples
+
+The model can be trained using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
+
+[#train-from-scratch-with-command-line-arguments-cli]
+=== Train from scratch with Command Line Arguments (CLI)
+
+Run the following command to train a model:
+
+[,sh]
+----
+pylaia-htr-train-ctc /path/to/syms.txt \
+ `cat img_dirs_args.txt`\
+ /path/to/train.txt \
+ /path/to/val.txt \
+ --trainer.gpus 1 \
+ --data.batch_size 32
+----
+
+[#train-from-scratch-with-a-yaml-configuration-file]
+=== Train from scratch with a YAML configuration file
+
+Run the following command to train a model:
+
+[,sh]
+----
+pylaia-htr-train-ctc --config config_train_model.yaml
+----
+
+Where `config_train_model.yaml` is:
+
+[,yaml]
+----
+syms: /path/to/syms.txt
+img_dirs:
+ - /path/to/images/
+tr_txt_table: /path/to/train.txt
+va_txt_table: /path/to/val.txt
+common:
+ experiment_dirname: experiment-dataset
+logging:
+ filepath: pylaia_training.log
+scheduler:
+ active: true
+train:
+ augment_training: true
+ early_stopping_patience: 80
+trainer:
+ auto_select_gpus: true
+ gpus: 1
+ max_epochs: 600
+----
+
+[#resume-training-from-a-checkpoint]
+=== Resume training from a checkpoint
+
+Run the following command to continue training from a checkpoint for 200 epochs.
+
+[,sh]
+----
+pylaia-htr-train-ctc --config config_train_model.yaml --train.resume true --trainer.max_epochs 200
+----
+
+NOTE: If `common.checkpoint` is not set, PyLaia will select the best checkpoint from `common.experiment_dirname`
+
+[#fine-tune-from-a-checkpoint]
+=== Fine-tune from a checkpoint
+
+Run the following command to load pretrained weights and fine-tune on a new dataset for 200 epochs.
+
+[,sh]
+----
+pylaia-htr-train-ctc --config config_train_model.yaml --common.experiment_dirname experiment/ --common.checkpoint initial_checkpoint.ckpt --train.pretrain true --trainer.max_epochs 200
+----
+
+[WARNING]
+// ====
+This option requires that your model architecture `model` matches the one used to train `initial_checkpoint.ckpt`.
+The last linear layer will be reinitialized using the Xavier initialization to match the new vocabulary size.
+// ====
+
+[NOTE]
+// ====
+The initial checkpoint is expected to be in the following directory: `{common.experiment_dirname}/pretrained/`.
+If it is located in `common.experiment_dirname`, the subdirectory `pretrained` will be created and the checkpoint will be moved there automatically.
+// ====
+
+[#train-on-right-to-left-reading-order]
+=== Train on Right-To-Left reading order
+
+By default, PyLaia expects images with Left-to-Right reading order.
+To train a model on Right-To-Left data, use the following command:
+
+[,sh]
+----
+pylaia-htr-train-ctc --config config_train_model_rtl.yaml
+----
+
+Where `config_train_model_rtl.yaml` is:
+
+[,yaml]
+----
+syms: /path/to/syms.txt
+img_dirs:
+ - /path/to/images/
+tr_txt_table: /path/to/train.txt
+va_txt_table: /path/to/val.txt
+common:
+ experiment_dirname: experiment-dataset
+logging:
+ filepath: pylaia_training.log
+scheduler:
+ active: true
+train:
+ augment_training: true
+ early_stopping_patience: 80
+trainer:
+ auto_select_gpus: true
+ gpus: 1
+ max_epochs: 600
+data:
+ reading_order: RTL
+----
+
+[#train-and-log-to-weights-biases]
+=== Train and log to Weights & Biases
+
+By default, PyLaia logs metrics and losses to a local CSV file. You can chose to log into https://wandb.ai/home[Weights & Biases] instead.
+
+To set up Weights & Biases:
+
+* Run `pip install pylaia[wandb]` to install the required dependencies
+* Sign in to Weights & Biases using `wandb login`
+
+Then, start training with `pylaia-htr-train-ctc --config config_train_model.yaml --train.log_to_wandb true`.
+
+This will create a project called `PyLaia` in W&B with one run for each training. The following are monitored for each run:
+
+* Training and validation metrics (losses, CER, WER)
+* Model gradients
+* System metrics (GPU and CPU utilisation, temperature, allocated memory)
+* Hyperparameters (training configuration)
+
+A public dashboard is available https://wandb.ai/starride-teklia/PyLaia%20demo[here] as an example.
diff --git a/docs/original_paper.md b/docs/original_paper.md
deleted file mode 100644
index b7133403136de97810ef4f43539794a4fce25a90..0000000000000000000000000000000000000000
--- a/docs/original_paper.md
+++ /dev/null
@@ -1,18 +0,0 @@
-# Original paper
-
-The original PyLaia model was presented in the paper entitled: [*Are Multidimensional Recurrent Layers Really Necessary for Handwritten Text Recognition?* from Joan Puigcerver, published in the 14th IAPR International Conference on Document Analysis and Recognition (ICDAR 2017)](https://ieeexplore.ieee.org/document/8269951).
-
-The full text is available on this [page](http://www.jpuigcerver.net/pubs/jpuigcerver_icdar2017.pdf).
-
-Recommended citation:
-```bibtex
-@INPROCEEDINGS{PyLaia,
- author={Puigcerver, Joan},
- booktitle={2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR)},
- title={Are Multidimensional Recurrent Layers Really Necessary for Handwritten Text Recognition?},
- year={2017},
- volume={01},
- number={},
- pages={67-72},
- doi={10.1109/ICDAR.2017.20}}
-```
diff --git a/docs/reference/callbacks/decode.md b/docs/reference/callbacks/decode.md
deleted file mode 100644
index 654ccf887d584720f71aa7331428803a38c56c1b..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/decode.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.decode
diff --git a/docs/reference/callbacks/learning_rate.md b/docs/reference/callbacks/learning_rate.md
deleted file mode 100644
index 0a0af35764edb744c4da3299aba2651f417918ed..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/learning_rate.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.learning_rate
diff --git a/docs/reference/callbacks/meters/meter.md b/docs/reference/callbacks/meters/meter.md
deleted file mode 100644
index 455b4011cb7de1b5a5e057f0b20d5be55c6df340..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/meters/meter.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.meters.meter
diff --git a/docs/reference/callbacks/meters/sequence_error.md b/docs/reference/callbacks/meters/sequence_error.md
deleted file mode 100644
index 51dce4f9f4f15e3a83cac4fbf84a5c692c7caeb3..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/meters/sequence_error.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.meters.sequence_error
diff --git a/docs/reference/callbacks/meters/timer.md b/docs/reference/callbacks/meters/timer.md
deleted file mode 100644
index 79e892b5de2f32ac2fbf88f4fe5e13557098da4b..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/meters/timer.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.meters.timer
diff --git a/docs/reference/callbacks/netout.md b/docs/reference/callbacks/netout.md
deleted file mode 100644
index 3ce68ab8be8215c456491fea4fdfa6a82e74d120..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/netout.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.netout
diff --git a/docs/reference/callbacks/progress_bar.md b/docs/reference/callbacks/progress_bar.md
deleted file mode 100644
index 251b23c56dfe59ffb31abd1914ef1748698274cf..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/progress_bar.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.progress_bar
diff --git a/docs/reference/callbacks/progress_bar_gpu_stats.md b/docs/reference/callbacks/progress_bar_gpu_stats.md
deleted file mode 100644
index 15cabb7d1b0a5d88d34b875c65daffd1735529c7..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/progress_bar_gpu_stats.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.progress_bar_gpu_stats
diff --git a/docs/reference/callbacks/segmentation.md b/docs/reference/callbacks/segmentation.md
deleted file mode 100644
index 1c1d58fe66ad48c6790e9ba1db0d6cda68285bf1..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/segmentation.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.segmentation
diff --git a/docs/reference/callbacks/training_timer.md b/docs/reference/callbacks/training_timer.md
deleted file mode 100644
index d7ec51f4ab28e0a50a1facd04b757fdecd731076..0000000000000000000000000000000000000000
--- a/docs/reference/callbacks/training_timer.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.callbacks.training_timer
diff --git a/docs/reference/common/arguments.md b/docs/reference/common/arguments.md
deleted file mode 100644
index 3a39fc7e5cf4a7b59f08490bc334d98fe2fec23f..0000000000000000000000000000000000000000
--- a/docs/reference/common/arguments.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.common.arguments
diff --git a/docs/reference/common/loader.md b/docs/reference/common/loader.md
deleted file mode 100644
index 83b47de0e450aa4ed831fdbd37d6c4a74d1a01a5..0000000000000000000000000000000000000000
--- a/docs/reference/common/loader.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.common.loader
diff --git a/docs/reference/common/logging.md b/docs/reference/common/logging.md
deleted file mode 100644
index 6d75685d93f9bb323e38c6fc37f6e79ae791e650..0000000000000000000000000000000000000000
--- a/docs/reference/common/logging.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.common.logging
diff --git a/docs/reference/common/saver.md b/docs/reference/common/saver.md
deleted file mode 100644
index 0c8aa3c2513c624260e24a19d568bc0a306d86b7..0000000000000000000000000000000000000000
--- a/docs/reference/common/saver.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.common.saver
diff --git a/docs/reference/common/types.md b/docs/reference/common/types.md
deleted file mode 100644
index 0947b2eff407108cfca28eabaaa2a7927c379107..0000000000000000000000000000000000000000
--- a/docs/reference/common/types.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.common.types
diff --git a/docs/reference/data/image_dataset.md b/docs/reference/data/image_dataset.md
deleted file mode 100644
index 8e935d1c37609164f2b45dd4b2a76e52d637650a..0000000000000000000000000000000000000000
--- a/docs/reference/data/image_dataset.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.image_dataset
diff --git a/docs/reference/data/image_from_list_dataset.md b/docs/reference/data/image_from_list_dataset.md
deleted file mode 100644
index 77cd2205e1d32db0c66c2556583f7d7c9cf414cb..0000000000000000000000000000000000000000
--- a/docs/reference/data/image_from_list_dataset.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.image_from_list_dataset
diff --git a/docs/reference/data/padding_collater.md b/docs/reference/data/padding_collater.md
deleted file mode 100644
index 8998b5c864197d50e907470e68c3d9d8fdc5db73..0000000000000000000000000000000000000000
--- a/docs/reference/data/padding_collater.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.padding_collater
diff --git a/docs/reference/data/text_image_dataset.md b/docs/reference/data/text_image_dataset.md
deleted file mode 100644
index 2dacfff6eed90ded1646b5ac24cc0e5f696f0628..0000000000000000000000000000000000000000
--- a/docs/reference/data/text_image_dataset.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.text_image_dataset
diff --git a/docs/reference/data/text_image_from_text_table_dataset.md b/docs/reference/data/text_image_from_text_table_dataset.md
deleted file mode 100644
index a7abfdb31af20650049e39db7b3a5d0ee331ff17..0000000000000000000000000000000000000000
--- a/docs/reference/data/text_image_from_text_table_dataset.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.text_image_from_text_table_dataset
diff --git a/docs/reference/data/transforms/text/transforms.md b/docs/reference/data/transforms/text/transforms.md
deleted file mode 100644
index 44a8d117c46c8c0105092ee58d1dec5928d31f06..0000000000000000000000000000000000000000
--- a/docs/reference/data/transforms/text/transforms.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.transforms.text.transforms
diff --git a/docs/reference/data/transforms/transforms.md b/docs/reference/data/transforms/transforms.md
deleted file mode 100644
index 663a8189013ec3bed798993e131e2eac537e2b34..0000000000000000000000000000000000000000
--- a/docs/reference/data/transforms/transforms.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.transforms.transforms
diff --git a/docs/reference/data/transforms/vision/random_beta_affine.md b/docs/reference/data/transforms/vision/random_beta_affine.md
deleted file mode 100644
index a1e7d9f32a5c968d62aa565af22364c75db6f4de..0000000000000000000000000000000000000000
--- a/docs/reference/data/transforms/vision/random_beta_affine.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.transforms.vision.random_beta_affine
diff --git a/docs/reference/data/transforms/vision/random_beta_morphology.md b/docs/reference/data/transforms/vision/random_beta_morphology.md
deleted file mode 100644
index 1b71a7ad00faf7532515e5476e4067585b86c676..0000000000000000000000000000000000000000
--- a/docs/reference/data/transforms/vision/random_beta_morphology.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.transforms.vision.random_beta_morphology
diff --git a/docs/reference/data/transforms/vision/random_beta_perspective.md b/docs/reference/data/transforms/vision/random_beta_perspective.md
deleted file mode 100644
index 1b3b7eb42967b7ea63d0646eecd0575030a70b89..0000000000000000000000000000000000000000
--- a/docs/reference/data/transforms/vision/random_beta_perspective.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.transforms.vision.random_beta_perspective
diff --git a/docs/reference/data/transforms/vision/transforms.md b/docs/reference/data/transforms/vision/transforms.md
deleted file mode 100644
index 6a1651448d77ada5982d38101557ef9b51dfb8ba..0000000000000000000000000000000000000000
--- a/docs/reference/data/transforms/vision/transforms.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.transforms.vision.transforms
diff --git a/docs/reference/data/unpadded_distributed_sampler.md b/docs/reference/data/unpadded_distributed_sampler.md
deleted file mode 100644
index 19018eab14603c69329c7c17dd189ebc110780db..0000000000000000000000000000000000000000
--- a/docs/reference/data/unpadded_distributed_sampler.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.data.unpadded_distributed_sampler
diff --git a/docs/reference/decoders/ctc_alignment.md b/docs/reference/decoders/ctc_alignment.md
deleted file mode 100644
index 29361041b6001d9bd89b168c21fcd0e49a96eb7e..0000000000000000000000000000000000000000
--- a/docs/reference/decoders/ctc_alignment.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.decoders.ctc_alignment
diff --git a/docs/reference/decoders/ctc_greedy_decoder.md b/docs/reference/decoders/ctc_greedy_decoder.md
deleted file mode 100644
index 257f14600b97a0112017adc76e861ba53b514f23..0000000000000000000000000000000000000000
--- a/docs/reference/decoders/ctc_greedy_decoder.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.decoders.ctc_greedy_decoder
diff --git a/docs/reference/decoders/ctc_language_decoder.md b/docs/reference/decoders/ctc_language_decoder.md
deleted file mode 100644
index 5c359ddee1fec5958192158e0065e61bc86ee7f6..0000000000000000000000000000000000000000
--- a/docs/reference/decoders/ctc_language_decoder.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.decoders.ctc_language_decoder
diff --git a/docs/reference/decoders/ctc_nbest_decoder.md b/docs/reference/decoders/ctc_nbest_decoder.md
deleted file mode 100644
index 7d7f86a804dead76e0f4b8fdbb71ca20ef687eac..0000000000000000000000000000000000000000
--- a/docs/reference/decoders/ctc_nbest_decoder.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.decoders.ctc_nbest_decoder
diff --git a/docs/reference/dummies/data_modules/dummy_mnist.md b/docs/reference/dummies/data_modules/dummy_mnist.md
deleted file mode 100644
index 56861584914ca49b68b80429fe02e7acd2dc743c..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/data_modules/dummy_mnist.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.data_modules.dummy_mnist
diff --git a/docs/reference/dummies/data_modules/dummy_mnist_lines.md b/docs/reference/dummies/data_modules/dummy_mnist_lines.md
deleted file mode 100644
index 2bb63744a20b1cd2f1c96852bbf391ee9b4ba74a..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/data_modules/dummy_mnist_lines.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.data_modules.dummy_mnist_lines
diff --git a/docs/reference/dummies/dummy_model.md b/docs/reference/dummies/dummy_model.md
deleted file mode 100644
index 7ad268b58f8526cba265a10ef8092155fed7a6a9..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/dummy_model.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.dummy_model
diff --git a/docs/reference/dummies/dummy_plugin.md b/docs/reference/dummies/dummy_plugin.md
deleted file mode 100644
index 56acbc00cccca22c738a48463d11c9baf70586e8..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/dummy_plugin.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.dummy_plugin
diff --git a/docs/reference/dummies/dummy_trainer.md b/docs/reference/dummies/dummy_trainer.md
deleted file mode 100644
index 09710003c45fe50d16166edf37054ba9631737d3..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/dummy_trainer.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.dummy_trainer
diff --git a/docs/reference/dummies/modules/dummy_engine.md b/docs/reference/dummies/modules/dummy_engine.md
deleted file mode 100644
index e7e261d7af615f0d611b8f36b2494ba0b21c2d17..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/modules/dummy_engine.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.modules.dummy_engine
diff --git a/docs/reference/dummies/modules/dummy_evaluator.md b/docs/reference/dummies/modules/dummy_evaluator.md
deleted file mode 100644
index ead6cac448ea4febe4e9c8c4b64efc287789d820..0000000000000000000000000000000000000000
--- a/docs/reference/dummies/modules/dummy_evaluator.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.dummies.modules.dummy_evaluator
diff --git a/docs/reference/engine/data_module.md b/docs/reference/engine/data_module.md
deleted file mode 100644
index 95e51e0928736151c040319e714b01afe03c5981..0000000000000000000000000000000000000000
--- a/docs/reference/engine/data_module.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine.data_module
diff --git a/docs/reference/engine/engine_exception.md b/docs/reference/engine/engine_exception.md
deleted file mode 100644
index ac3fa80482d80a4ed699345a9835d090c21b4540..0000000000000000000000000000000000000000
--- a/docs/reference/engine/engine_exception.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine.engine_exception
diff --git a/docs/reference/engine/engine_module.md b/docs/reference/engine/engine_module.md
deleted file mode 100644
index 8ece90bec2c5f466b0e5693e0d1a72386d506766..0000000000000000000000000000000000000000
--- a/docs/reference/engine/engine_module.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine.engine_module
diff --git a/docs/reference/engine/evaluator_module.md b/docs/reference/engine/evaluator_module.md
deleted file mode 100644
index 9f9141683dd7dbd7300fb49c64acda5d5d7c3337..0000000000000000000000000000000000000000
--- a/docs/reference/engine/evaluator_module.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine.evaluator_module
diff --git a/docs/reference/engine/feeder.md b/docs/reference/engine/feeder.md
deleted file mode 100644
index fe848e055e67450c66e1e99a903cd8e6b6e5f92a..0000000000000000000000000000000000000000
--- a/docs/reference/engine/feeder.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine.feeder
diff --git a/docs/reference/engine/htr_engine_module.md b/docs/reference/engine/htr_engine_module.md
deleted file mode 100644
index cf412c47afacac423518daa8b7fa80a03def5aba..0000000000000000000000000000000000000000
--- a/docs/reference/engine/htr_engine_module.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine.htr_engine_module
diff --git a/docs/reference/engine/index.md b/docs/reference/engine/index.md
deleted file mode 100644
index 05a8adcc47d4a735f6ec4c863faf2b30b25f4d12..0000000000000000000000000000000000000000
--- a/docs/reference/engine/index.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.engine
diff --git a/docs/reference/loggers/epoch_csv_logger.md b/docs/reference/loggers/epoch_csv_logger.md
deleted file mode 100644
index f1c9f10fcd4eb9f26ba5d5eec60b41b14526bd27..0000000000000000000000000000000000000000
--- a/docs/reference/loggers/epoch_csv_logger.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.loggers.epoch_csv_logger
diff --git a/docs/reference/losses/ctc_loss.md b/docs/reference/losses/ctc_loss.md
deleted file mode 100644
index 1763816baaac582afc166eb636c1f4c00ad1a700..0000000000000000000000000000000000000000
--- a/docs/reference/losses/ctc_loss.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.losses.ctc_loss
diff --git a/docs/reference/losses/loss.md b/docs/reference/losses/loss.md
deleted file mode 100644
index 207b0c1cb5b204d46076b721183c9ddb0988f5af..0000000000000000000000000000000000000000
--- a/docs/reference/losses/loss.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.losses.loss
diff --git a/docs/reference/models/htr/conv_block.md b/docs/reference/models/htr/conv_block.md
deleted file mode 100644
index 293a8ba548f49c4f99fcea6fca45bedd3f6f96ae..0000000000000000000000000000000000000000
--- a/docs/reference/models/htr/conv_block.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.models.htr.conv_block
diff --git a/docs/reference/models/htr/gated_crnn.md b/docs/reference/models/htr/gated_crnn.md
deleted file mode 100644
index 696ccd0987fd0b79e8ed93d8307823691e46243e..0000000000000000000000000000000000000000
--- a/docs/reference/models/htr/gated_crnn.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.models.htr.gated_crnn
diff --git a/docs/reference/models/htr/laia_crnn.md b/docs/reference/models/htr/laia_crnn.md
deleted file mode 100644
index 942ba12a519100116a580d618f5e29d9313c9df8..0000000000000000000000000000000000000000
--- a/docs/reference/models/htr/laia_crnn.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.models.htr.laia_crnn
diff --git a/docs/reference/models/index.md b/docs/reference/models/index.md
deleted file mode 100644
index af6229fa43f2a554c03396bb122526e66e598174..0000000000000000000000000000000000000000
--- a/docs/reference/models/index.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.models
diff --git a/docs/reference/nn/adaptive_pool_2d.md b/docs/reference/nn/adaptive_pool_2d.md
deleted file mode 100644
index 0aa6f776255df75fcc1613d8e9a080c9df871ac1..0000000000000000000000000000000000000000
--- a/docs/reference/nn/adaptive_pool_2d.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.nn.adaptive_pool_2d
diff --git a/docs/reference/nn/image_pooling_sequencer.md b/docs/reference/nn/image_pooling_sequencer.md
deleted file mode 100644
index 46a7404cac07ac70c0e863ee2568087e8cf49721..0000000000000000000000000000000000000000
--- a/docs/reference/nn/image_pooling_sequencer.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.nn.image_pooling_sequencer
diff --git a/docs/reference/nn/image_to_sequence.md b/docs/reference/nn/image_to_sequence.md
deleted file mode 100644
index 7eb166bc658d2cef67176b5ee19a89e9ef904608..0000000000000000000000000000000000000000
--- a/docs/reference/nn/image_to_sequence.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.nn.image_to_sequence
diff --git a/docs/reference/nn/pyramid_maxpool_2d.md b/docs/reference/nn/pyramid_maxpool_2d.md
deleted file mode 100644
index b11c9470ad2c2a862883594985f1c2f30f36e8df..0000000000000000000000000000000000000000
--- a/docs/reference/nn/pyramid_maxpool_2d.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.nn.pyramid_maxpool_2d
diff --git a/docs/reference/nn/resnet.md b/docs/reference/nn/resnet.md
deleted file mode 100644
index 4be46ba0e9028361563e9a6238df80c1c8aff812..0000000000000000000000000000000000000000
--- a/docs/reference/nn/resnet.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.nn.resnet
diff --git a/docs/reference/nn/temporal_pyramid_maxpool_2d.md b/docs/reference/nn/temporal_pyramid_maxpool_2d.md
deleted file mode 100644
index 7b99bfcae083d125f4053ab3f548bfcf7b7d6a27..0000000000000000000000000000000000000000
--- a/docs/reference/nn/temporal_pyramid_maxpool_2d.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.nn.temporal_pyramid_maxpool_2d
diff --git a/docs/reference/scripts/htr/create_model.md b/docs/reference/scripts/htr/create_model.md
deleted file mode 100644
index 436a7738d796bb3835efb68af106dde9ebacecfd..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/htr/create_model.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts.htr.create_model
diff --git a/docs/reference/scripts/htr/dataset/index.md b/docs/reference/scripts/htr/dataset/index.md
deleted file mode 100644
index 453b015232110903f558c6276c71baeb8318dded..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/htr/dataset/index.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts.htr.dataset
diff --git a/docs/reference/scripts/htr/dataset/validate.md b/docs/reference/scripts/htr/dataset/validate.md
deleted file mode 100644
index 68d92fc17072615802af13d96b397f1cabdda157..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/htr/dataset/validate.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts.htr.dataset.validate
diff --git a/docs/reference/scripts/htr/decode_ctc.md b/docs/reference/scripts/htr/decode_ctc.md
deleted file mode 100644
index 5189973f307c70dac9bc52c7ece8e08b8dd56151..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/htr/decode_ctc.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts.htr.decode_ctc
diff --git a/docs/reference/scripts/htr/netout.md b/docs/reference/scripts/htr/netout.md
deleted file mode 100644
index f8c64c2073dfef71db0fe3d96fefb6d42c6fbb3a..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/htr/netout.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts.htr.netout
diff --git a/docs/reference/scripts/htr/train_ctc.md b/docs/reference/scripts/htr/train_ctc.md
deleted file mode 100644
index 9b900b88a8defc6a6044a73c93b4cccfcfffd876..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/htr/train_ctc.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts.htr.train_ctc
diff --git a/docs/reference/scripts/index.md b/docs/reference/scripts/index.md
deleted file mode 100644
index 4e6118b19dc58a4eed8c1ea5e64e55f181e0f437..0000000000000000000000000000000000000000
--- a/docs/reference/scripts/index.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.scripts
diff --git a/docs/reference/utils/checks.md b/docs/reference/utils/checks.md
deleted file mode 100644
index b32df6810849e763e42a3cb8dfae8cc31fd73779..0000000000000000000000000000000000000000
--- a/docs/reference/utils/checks.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.utils.checks
diff --git a/docs/reference/utils/kaldi.md b/docs/reference/utils/kaldi.md
deleted file mode 100644
index bf088688d13949b13f81f881d173aeffc5192555..0000000000000000000000000000000000000000
--- a/docs/reference/utils/kaldi.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.utils.kaldi
diff --git a/docs/reference/utils/mdutils.md b/docs/reference/utils/mdutils.md
deleted file mode 100644
index 126b74b142acf42dc4d9861df33e2e6ec821bf89..0000000000000000000000000000000000000000
--- a/docs/reference/utils/mdutils.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.utils.mdutils
diff --git a/docs/reference/utils/stats.md b/docs/reference/utils/stats.md
deleted file mode 100644
index 1da41c795d29d9129895f7ca7024246d6a1fd8b0..0000000000000000000000000000000000000000
--- a/docs/reference/utils/stats.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.utils.stats
diff --git a/docs/reference/utils/symbols_table.md b/docs/reference/utils/symbols_table.md
deleted file mode 100644
index 188abcf689a1e2aa3916c6fe153efa5712f1357a..0000000000000000000000000000000000000000
--- a/docs/reference/utils/symbols_table.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.utils.symbols_table
diff --git a/docs/reference/utils/visualize_segmentation.md b/docs/reference/utils/visualize_segmentation.md
deleted file mode 100644
index 8e689434116b9a8c074b4f1a2163dc52d5c2add2..0000000000000000000000000000000000000000
--- a/docs/reference/utils/visualize_segmentation.md
+++ /dev/null
@@ -1 +0,0 @@
-::: laia.utils.visualize_segmentation
diff --git a/docs/releases.md b/docs/releases.md
deleted file mode 100644
index b3adea9fef67fbe81f5623e67d0812ce0966dbb7..0000000000000000000000000000000000000000
--- a/docs/releases.md
+++ /dev/null
@@ -1,205 +0,0 @@
-# Releases
-
-## 1.1.1
-
-Released on **12 August 2024** • View on [Gitlab](https://gitlab.teklia.com/atr/pylaia/-/releases/1.1.1)
-
-### Breaking changes
-
-- The [nnutils](https://gitlab.teklia.com/atr/nnutils/) library is no longer maintained and is only compatible with Python 3.6, 3.7, 3.8. As such its dependency has been removed. The `crnn.use_masks` parameter has been removed. It is still supported to keep the compatibility with older training configuration but will be ignored.
-
-### Feature
-
-- The number of worker processes created in dataloaders is now exposed through the `data.num_workers` parameter.
-- There is a new command to run basic checks and compute statistics on your training dataset. Learn more about it in [the documentation](https://atr.pages.teklia.com/pylaia/usage/datasets/).
-- Pretraining is now available. Load the weights of a previous checkpoint using the `train.pretrain` parameter when fine-tuning a model on a new dataset. Learn more about it in [the documentation](https://atr.pages.teklia.com/pylaia/usage/training/#resume-training-from-a-checkpoint).
-- When training on a small dataset, freezing some of the layers can help with model convergence. The `train.freeze_layers` parameter supports freezing:
-
- - convolutional layers,
- - recurrent layers,
- - linear layers.
-
-- Proper support for right-to-left (RTL) languages is now available. Enable it using the `data.reading_order` argument both during [training](https://atr.pages.teklia.com/pylaia/usage/training/#train-on-right-to-left-reading-order) and [decoding](https://atr.pages.teklia.com/pylaia/usage/prediction/#predict-on-right-to-left-data).
-
-### Dependencies
-
-- Bumped [pytorch-lightning](https://pypi.org/project/pytorch-lightning/) to version `1.4.2`.
-- Bumped [textdistance](https://pypi.org/project/textdistance/) to version `4.6.1`.
-
-### Misc
-
-- A deprecation warning from jsonargparse was fixed.
-- The package's metadata are now stored in `pyproject.toml` as per [PEP-0621](https://peps.python.org/pep-0621/).
-- PyLaia now uses [ruff](https://docs.astral.sh/ruff/) for linting and formatting.
-
-## 1.1.0
-
-Released on **22 December 2023** • View on [Gitlab](https://gitlab.teklia.com/atr/pylaia/-/releases/1.1.0)
-
-### Breaking changes
-
-- Official support for Python3.8 has been dropped. This doesn't mean that the current code doesn't run on python3.8, we simply do not test that compatibility anymore. This decision was made since active support of python 3.8 has stopped for a while now and many libraries in the ML world have stopped supporting it as well.
-
-### Feature
-
-- A Docker image with the needed code to use this library is now built on every tag.
-- The coverage of our tests suite is displayed again as a GitLab badge on the repository as well as in the README.md file.
-
-### Documentation
-
-- Many sections were added to the documentation:
-
- - for the [pylaia-htr-create-model](https://atr.pages.teklia.com/pylaia/usage/initialization/) command,
- - for [dataset formatting](https://atr.pages.teklia.com/pylaia/usage/datasets/),
- - for the [pylaia-htr-train-ctc](https://atr.pages.teklia.com/pylaia/usage/training/) command and [fine-tuning](https://atr.pages.teklia.com/pylaia/usage/training/#resume-training-from-a-checkpoint),
- - for the [pylaia-htr-decode-ctc](https://atr.pages.teklia.com/pylaia/usage/prediction/) command,
- - for the [pylaia-htr-netout](https://atr.pages.teklia.com/pylaia/usage/netout/) command,
- - to [train](https://atr.pages.teklia.com/pylaia/usage/language_models/) [KenLM](https://kheafield.com/code/kenlm/) language models,
- - the full Python code reference.
-
-- A contribution guide and a code of conduct were added for new contributors.
-
-### Dependencies
-
-- Bumped [pytorch-lightning](https://pypi.org/project/pytorch-lightning/) to version `1.3.0`
-- Some dependencies were pinned to a version to avoid breakage:
-
- - [natsort](https://pypi.org/project/natsort/) was pinned to version `8.4.0`,
- - [textdistance](https://pypi.org/project/textdistance/) was pinned to version `4.6.0`,
- - [scipy](https://pypi.org/project/scipy/) was pinned to version `1.11.3`,
- - [matplotlib](https://pypi.org/project/matplotlib/) was pinned to version `3.8.2`,
- - [numpy](https://pypi.org/project/numpy/) direct dependency was removed since it's installed through `scipy` and `matplotlib`.
-
-- PyLaia dropped support for python 3.8 so the [dataclasses](https://pypi.org/project/dataclasses/) dependency was dropped.
-
-### Misc
-
-- The `torch.testing.assert_allclose` has been replaced by `torch.testing.assert_close` since it became deprecated in [PyTorch 1.12.0](https://github.com/pytorch/pytorch/issues/61844).
-
-
-## 1.0.7
-
-Released on **18 October 2023** • View on [Gitlab](https://gitlab.teklia.com/atr/pylaia/-/releases/1.0.7)
-
-### Feature
-- When using a language model, a confidence score is now returned based on the log-likelyhood of the hypothesis.
-
-### Documentation
-A public documentation is now available on <https://atr.pages.teklia.com/pylaia/>. It's still under construction but next releases will add more and more content.
-
-### Dependencies
-- Bumped [pytorch-lightning](https://pypi.org/project/pytorch-lightning/) to version `1.1.7`
-- Bumped GitHub action [codecov/codecov-action](https://github.com/codecov/codecov-action) to version `3`
-- Bumped GitHub action [actions/setup-python](https://github.com/actions/setup-python) to version `4`
-- Bumped GitHub action [actions/checkout](https://github.com/actions/checkout) to version `4`
-
-### Development
-- Releases are now built more easily through a Makefile.
-- The documentation is also redeployed after each push on `master` branch.
-- Fixed a test that behaved differently locally and during CI.
-
-## 1.0.6
-
-Released on **12 September 2023** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.6)
-
-### Feature
-- During training, too small images are now padded to be able to pass the multiple convolution layers.
-
-### Documentation
-- Fixed typos.
-
-### Dependencies
-- Replaced [deprecated Pillow resampling method](https://pillow.readthedocs.io/en/stable/releasenotes/2.7.0.html#antialias-renamed-to-lanczos) `Image.ANTIALIAS` to `Image.Resample.Lanczos`.
-
-### Development
-- Pre-commit hooks were updated.
-
-## 1.0.5
-
-Released on **29 March 2023** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.5)
-
-### Dependencies
-- Requires `torch` version `1.13.0` or `1.13.1`.
-- Requires `torchvision` version `0.14.0` or `0.14.1` (depending on `torch` version).
-
-## 1.0.4
-
-Released on **4 January 2023** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.4)
-
-### Dependencies
-- Requires `torch` version `1.13.0`.
-
-## 1.0.3
-
-Released on **12 December 2022** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.3)
-
-### Feature
-- Now able to decode using a trained Language model through beam search decoding.
-- Exposes [torch Dataloaders's num_workers](https://pytorch.org/docs/stable/data.html#multi-process-data-loading) parameter on the Python training function to limit resource usage when needed.
-
-### Dependencies
-- Added dependency to `torchaudio` version `0.13.0`.
-
-### Development
-- Package version is now tracked through the `VERSION` file.
-
-## 1.0.2
-
-Released on **7 December 2022** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.2)
-
-### Dependencies
-- Pinned dependency to `pytorch-lightning` to version `1.1.0`.
-
-## 1.0.1
-
-Released on **7 December 2022** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.1)
-
-## 1.0.0
-
-Released on **2 December 2020** • View on [Github](https://github.com/jpuigcerver/PyLaia/releases/tag/1.0.0)
-
-### Added
-
-- Support distributed training
-- Scripts can now be configured using yaml configuration files
-- Add support for the SGD and Adam optimizers
-- Support color images
-- Log the installed version of each module when scripts are called from shell
-- Add char/word segmentation to the decode script
-- Add several badges to the README
-- Support using a `ReduceLROnPlateau` scheduler during training
-- A CSV file (metrics.csv) is now created with the results obtained during training
-- Add CONTRIBUTING file
-- Training now can include GPU stats in the progress bar
-- Add isort to pre-commit to keep consistent imports throughout the codebase
-- Users can run the PyLaia scripts using Python now
-- Support half-precision training for fixed height models.
-- Add script to visualize the segmentation output
-- Use Codecov to produce test coverage reports
-- Code is now analyzed using CodeFactor
-
-### Changed
-
-- Make Python 3.6 the minimum supported version
-- Make PyTorch 1.4.0 the minimum supported version
-- Remove `ImageToTensor` in favor of vision transform `ToImageTensor`
-- Remove all of the internal logic (`engine`, `actions`, `hooks`, etc) in favor of pytorch-lightning's constructs
-- Change Travis CI for GitHub actions
-- Greatly improve the progress bar. It is used now in all scripts
-- The entire shell API has changed for the better (thanks to jsonargparse). Arguments are now separated into groups and help messages are clearer.
-- Drastically improve our test suite, we now have a 91% coverage
-
-### Removed
-
-- Remove egs directory. These live now at https://github.com/carmocca/PyLaia-examples
-- Remove Baidu's CTC loss in favor of PyTorch's
-- Remove PHOC code. Please open an issue if you were using it
-- Remove Dortmund code. Please open an issue if you were using it
-- Remove CTCLatticeGenerator. Please open an issue if you were using it
-- We no longer support saving checkpoints for more than one metric. Will be added back in a future version
-
-### Fixed
-
-- Fix WER calculation when long delimiters are used
-- Exit training if a delimiter is not present in the vocabulary
-- Hundreds of other minor fixes and refactors to improve the code quality!
diff --git a/docs/ui/partials/footer-content.hbs b/docs/ui/partials/footer-content.hbs
new file mode 100644
index 0000000000000000000000000000000000000000..21c3b010b529f8d554d92c973777a78effc7e82c
--- /dev/null
+++ b/docs/ui/partials/footer-content.hbs
@@ -0,0 +1,3 @@
+<footer class="footer">
+ <p>Copyright © <a href="https://teklia.com" target="_blank">Teklia</a>
+</footer>
diff --git a/docs/ui/partials/header-content.hbs b/docs/ui/partials/header-content.hbs
new file mode 100644
index 0000000000000000000000000000000000000000..4782603f23f8365791b5bf3e346cb4cd89dd5c5c
--- /dev/null
+++ b/docs/ui/partials/header-content.hbs
@@ -0,0 +1,32 @@
+<header class="header">
+ <nav class="navbar">
+ <div class="navbar-brand">
+ <div class="navbar-item">
+ <a href="{{or site.url (or siteRootUrl siteRootPath)}}">{{ site.title }}</a>
+ </div>
+ {{#if env.SITE_SEARCH_PROVIDER}}
+ <div class="navbar-item search hide-for-print">
+ <div id="search-field" class="field">
+ <input id="search-input" type="text" placeholder="Search the docs"{{#if page.home}} autofocus{{/if}}>
+ </div>
+ </div>
+ {{/if}}
+ <button class="navbar-burger" data-target="topbar-nav">
+ <span></span>
+ <span></span>
+ <span></span>
+ </button>
+ </div>
+ <div id="topbar-nav" class="navbar-menu">
+ <div class="navbar-end">
+ <div class="navbar-item has-dropdown is-hoverable">
+ <span class="navbar-link">Community</span>
+ <div class="navbar-dropdown">
+ <a class="navbar-item" target="_blank" href="https://gitlab.teklia.com/atr/pylaia/">Contribute</a>
+ <a class="navbar-item" target="_blank" href="https://support.teklia.com/c/machine-learning/pylaia/13">Support forum</a>
+ </div>
+ </div>
+ </div>
+ </div>
+ </nav>
+</header>
diff --git a/docs/usage/index.md b/docs/usage/index.md
deleted file mode 100644
index 3f14c12b1962240d041ae2f35e6dac7f94966725..0000000000000000000000000000000000000000
--- a/docs/usage/index.md
+++ /dev/null
@@ -1,20 +0,0 @@
-# Usage
-
-Once the dataset is formatted and `pylaia` is installed and in your environment, you may use the following commands:
-
-* `pylaia-htr-create-model`
-: To create a new PyLaia model. More details in the [dedicated page](./initialization/index.md).
-* `pylaia-htr-dataset-validate`
-: To compute statistics and run validation checks on a dataset. More details in the [dedicated page](./datasets/index.md).
-* `pylaia-htr-train-ctc`
-: To train a PyLaia model. More details in the [dedicated page](./training/index.md).
-* `pylaia-htr-decode-ctc`
-: To predict using a trained PyLaia model. More details in the [dedicated page](./prediction/index.md).
-* `pylaia-htr-netout`
-: To dump features from a PyLaia model. More details in the [dedicated page](./netout/index.md).
-
----
-Related pages:
-
-* Learn how to format a [dataset in PyLaia format](./datasets/format.md)
-* Learn how to use PyLaia with an [explicit language model](./language_models/index.md)
diff --git a/docs/usage/initialization/index.md b/docs/usage/initialization/index.md
deleted file mode 100644
index d55a25519419ac56db1975071dbac0f0b5584cd4..0000000000000000000000000000000000000000
--- a/docs/usage/initialization/index.md
+++ /dev/null
@@ -1,136 +0,0 @@
-# Model initialization
-
-The `pylaia-htr-create-model` command can be used to create a PyLaia model. To know more about the options of this command, use `pylaia-htr-create-model --help`.
-
-## Purpose
-
-The general architecture of PyLaia is composed of convolutional blocks followed by a set a bi-directionnal recurrent layers and a linear layer. PyLaia is fully configurable by the user, including:
-
-- Number of convolutional blocks,
-- Number of recurrent layers,
-- Batch normalization,
-- Pooling layers,
-- Activation function,
-- ...
-
-This command will create a pickled file (named `model` by default), which is required to initialize the `LaiaCRNN` class before training.
-
-## Parameters
-
-The full list of parameters is detailed in this section.
-
-
-### General parameters
-
-| Parameter | Description | Type | Default |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------------ |
-| `syms` | Positional argument. Path to a file mapping characters to integers. The CTC symbol must be mapped to integer 0. | `str` | |
-| `config` | Path to a JSON configuration file | `json` | |
-| `fixed_input_height` | Height of the input images. If set to 0, a variable height model will be used (see `adaptive_pooling`). This will be used to compute the model output height at the end of the convolutional layers. | `int` | 0 |
-| `adaptive_pooling` | Use custom adaptive pooling layers to enable training with variable height images. Takes into account the size of each individual image within the batch (before padding). Should be in `{avg,max}pool-N`. | `str` | `avgpool-16` |
-| `save_model` | Whether to save the model to a file. | `bool` | `True` |
-
-### Common parameters
-
-| Name | Description | Type | Default |
-| ----------------------- | --------------------------------------- | ----- | ------- |
-| `common.train_path` | Directory where the model will be saved | `str` | `.` |
-| `common.model_filename` | Filename of the model. | `str` | `model` |
-
-### Logging arguments
-
-| Name | Description | Type | Default |
-| ------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------- |
-| `logging.fmt` | Logging format. | `str` | `%(asctime)s %(levelname)s %(name)s] %(message)s` |
-| `logging.level` | Logging level. Should be in `{NOTSET,DEBUG,INFO,WARNING,ERROR,CRITICAL}` | `Level` | `INFO` |
-| `logging.filepath` | Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname` | `Optional[str]` | |
-| `logging.overwrite` | Whether to overwrite the logfile or to append. | `bool` | `False` |
-| `logging.to_stderr_level` | If filename is set, use this to log also to stderr at the given level. | `Level` | `ERROR` |
-
-### Architecture arguments
-
-
-| Name | Description | Type | Default |
-| ------------------------- | --------------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------ |
-| `crnn.num_input_channels` | Number of channels of the input images. | `int` | `1` |
-| `crnn.vertical_text` | Whether the text is written vertically. | `bool` | `False` |
-| `crnn.cnn_num_features` | Number of features in each convolutional layer. | `List` | `[16, 16, 32, 32]` |
-| `crnn.cnn_kernel_size` | Kernel size of each convolutional layer (e.g. [n,n,...] or [[h1,w1],[h2,w2],...]). | `List` | `[3, 3, 3, 3]` |
-| `crnn.cnn_stride` | Stride of each convolutional layer. (e.g. [n,n,...] or [[h1,w1],[h2,w2],...]) | `List` | `[1, 1, 1, 1]` |
-| `crnn.cnn_dilation` | Spacing between each convolutional layer kernel elements. (e.g. [n,n,...] or [[h1,w1],[h2,w2],...]) | `List` | `[1, 1, 1, 1]` |
-| `crnn.cnn_activation` | Type of activation function in each convolutional layer (from `torch.nn`). | `List` | `['LeakyReLU', 'LeakyReLU', 'LeakyReLU', 'LeakyReLU']` |
-| `crnn.cnn_poolsize` | MaxPooling size after each convolutional layer. (e.g. [n,n,...] or [[h1,w1],[h2,w2],...]). | `List` | `[2, 2, 2, 0]` |
-| `crnn.cnn_dropout` | Dropout probability at the input of each convolutional layer. | `List` | `[0.0, 0.0, 0.0, 0.0]` |
-| `crnn.cnn_batchnorm` | Whether to do batch normalization before the activation in each convolutional layer. | `List` | `[False, False, False, False]` |
-| `crnn.rnn_layers` | Number of recurrent layers. | `int` | `3` |
-| `crnn.rnn_units` | Number of units in each recurrent layer. | `int` | `256` |
-| `crnn.rnn_dropout` | Dropout probability at the input of each recurrent layer. | `float` | `0.5` |
-| `crnn.rnn_type` | Type of recurrent layer (from `torch.nn`). | `str` | `LSTM` |
-| `crnn.lin_dropout` | Dropout probability at the input of the final linear layer. | `float` | `0.5` |
-
-## Examples
-
-The model can be configured using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
-
-
-### Example with Command Line Arguments (CLI)
-
-Run the following command to create a model:
-```sh
-pylaia-htr-create-model /path/to/syms.txt \
- --fixed_input_height 128 \
- --crnn.rnn_layers 4 \
- --logging.filepath model.log \
- --common.train_path my_experiments/
-```
-
-### Example with a YAML configuration file
-
-Run the following command to create a model:
-```sh
-pylaia-htr-create-model --config config_create_model.yaml
-```
-
-Where `config_create_model.yaml` is:
-
-```yaml
-crnn:
- cnn_activation:
- - LeakyReLU
- - LeakyReLU
- - LeakyReLU
- - LeakyReLU
- cnn_batchnorm:
- - true
- - true
- - true
- - true
- cnn_dilation:
- - 1
- - 1
- - 1
- - 1
- cnn_kernel_size:
- - 3
- - 3
- - 3
- - 3
- cnn_num_features:
- - 12
- - 24
- - 48
- - 48
- cnn_poolsize:
- - 2
- - 2
- - 0
- - 2
- lin_dropout: 0.5
- rnn_dropout: 0.5
- rnn_layers: 3
- rnn_type: LSTM
- rnn_units: 256
-fixed_input_height: 128
-save_model: true
-syms: /path/to/syms.txt
-```
diff --git a/docs/usage/netout/index.md b/docs/usage/netout/index.md
deleted file mode 100644
index 039c3a903e964e71100a97bdb80f08f3a8476ffb..0000000000000000000000000000000000000000
--- a/docs/usage/netout/index.md
+++ /dev/null
@@ -1,137 +0,0 @@
-# Netout
-
-The `pylaia-htr-netout` command can be used to dump the features extracted by PyLaia for a set of text-lines. To know more about the options of this command, use `pylaia-htr-netout --help`.
-
-!!! warning
- This command was initially designed to combine PyLaia and Kaldi. Since December 2022, combining PyLaia with language models can be achieved more easily by [building a language model with KenLM](../language_models/index.md) and [predicting with `pylaia-htr-decode-ctc`](../prediction/index.md#predict-with-a-language-model).
-
-## Purpose
-
-This command outputs the feature matrix and lattice computed by PyLaia in Kaldi format for a given dataset.
-
-It requires:
-
-- a [list of image ids](../datasets/index.md#image-names),
-- the pickled `model` file created during [model initialization](../initialization/index.md),
-- the weights `*.ckpt` of the trained model created during [model training](../training/index.md).
-
-The files generated by this command are designed to combine PyLaia and Kaldi, but could also be used to predict with a custom decoder.
-
-## Parameters
-
-The full list of parameters is detailed in this section.
-
-### General parameters
-
-| Parameter | Description | Type | Default |
-| ---------- | -------------------------------------------------------------------------------------------- | ------ | ------- |
-| `img_list` | Positional argument. File containing the names of the images to decode (one image per line). | `str` | |
-| `img_dirs` | Directories containing line images. | `str` | |
-| `config` | Path to a JSON configuration file | `json` | |
-
-### Common parameters
-
-| Name | Description | Type | Default |
-| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | ------- |
-| `common.train_path` | Directory where the model will be saved | `str` | `.` |
-| `common.model_filename` | Filename of the model. | `str` | `model` |
-| `common.experiment_dirname` | Directory name of the experiment. | `experiment` | `74565` |
-| `common.checkpoint` | Checkpoint to load. Must be a filepath, a filename, a glob pattern or `None` (in this case, the best checkpoint will be loaded). Note that the checkpoint will be searched in `common.experiment_dirname`, unless you provide an absolute filepath. | `int` | `None` |
-
-### Data arguments
-
-| Name | Description | Type | Default |
-| ----------------- | ------------------------------------------------ | ----------- | ------------- |
-| `data.batch_size` | Batch size. | `int` | `8` |
-| `data.color_mode` | Color mode. Must be either `L`, `RGB` or `RGBA`. | `ColorMode` | `ColorMode.L` |
-
-### Netout arguments
-
-| Name | Description | Type | Default |
-| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | ------- |
-| `netout.output_transform` | Transformation to apply at the end of the model. Should be `softmax` or `log_softmax`. | `str` | `None` |
-| `netout.matrix` | Path to the output file containing a list of keys (image ids) and values (output matrix where rows represents timesteps and columns CTC labels). This file can be directly used with Kaldi. | `Optional[str]` | `None` |
-| `netout.lattice` | Path to the output file containing containing a list of keys (image ids) and values (lattices representing the CTC output). This file can be directly used with Kaldi. | `Optional[str]` | `None` |
-| `netout.digits` | Number of digits to be used for formatting | `int` | `10` |
-
-### Logging arguments
-
-| Name | Description | Type | Default |
-| ------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------- |
-| `logging.fmt` | Logging format. | `str` | `%(asctime)s %(levelname)s %(name)s] %(message)s` |
-| `logging.level` | Logging level. Should be in `{NOTSET,DEBUG,INFO,WARNING,ERROR,CRITICAL}` | `Level` | `INFO` |
-| `logging.filepath` | Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname` | `Optional[str]` | |
-| `logging.overwrite` | Whether to overwrite the logfile or to append. | `bool` | `False` |
-| `logging.to_stderr_level` | If filename is set, use this to log also to stderr at the given level. | `Level` | `ERROR` |
-
-### Trainer arguments
-
-Pytorch Lightning `Trainer` flags can also be set using the `--trainer` argument. See [the documentation](https://github.com/Lightning-AI/lightning/blob/1.7.0/docs/source-pytorch/common/trainer.rst#trainer-flags).
-
-This flag is mostly useful to define whether to predict on CPU or GPU.
-
-* `--trainer.gpus 0` to run on CPU,
-* `--trainer.gpus n` to run on `n` GPUs (use with `--training.auto_select True` for auto-selection),
-* `--trainer.gpus -1` to run on all GPUs.
-
-
-## Examples
-
-Dumping PyLaia's features can be done using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
-
-### Dumping features from a model from Hugging Face
-
-First, clone a trained model from Hugging Face:
-```bash
-git clone https://huggingface.co/Teklia/pylaia-huginmunin
-```
-
-List image names in `img_list.txt`:
-```text title="img_list.txt"
-docs/assets/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f
-docs/assets/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4
-```
-
-Dump PyLaia's output with:
-```bash
-pylaia-htr-netout --common.experiment_dirname pylaia-huginmunin/ \
- --common.model_filename pylaia-huginmunin/model \
- --netout.matrix matrix.txt \
- --netout.lattice lattice.txt \
- --img_dir [docs/assets] \
- img_list.txt
-```
-
-Output files will be written in `--common.experiment_dirname`:
-```
-├── pylaia-huginmunin/
- ├── matrix.txt
- └── lattice.txt
-```
-
-### Dumping features using a YAML configuration file
-
-Run the following command to dump PyLaia's output:
-```bash
-pylaia-htr-decode-ctc --config config_netout.yaml
-```
-
-With the following configuration file:
-```yaml title="config_netout.yaml"
-common:
- experiment_dirname: pylaia-huginmunin
- model_filename: pylaia-huginmunin/model
-img_list: img_list.txt
-img_dirs:
- - docs/assets/
-netout:
- matrix: matrix.txt
- lattice: lattice.txt
-```
-
-Output files will be written in `--common.experiment_dirname`:
-```
-├── pylaia-huginmunin/
- ├── matrix.txt
- └── lattice.txt
-```
diff --git a/docs/usage/prediction/index.md b/docs/usage/prediction/index.md
deleted file mode 100644
index 134255a4b65ed1b4e72a7f19da626bc44f1bf124..0000000000000000000000000000000000000000
--- a/docs/usage/prediction/index.md
+++ /dev/null
@@ -1,317 +0,0 @@
-# Decoding
-
-The `pylaia-htr-decode-ctc` command can be used to predict using a trained PyLaia model. To know more about the options of this command, use `pylaia-htr-decode-ctc --help`.
-
-## Purpose
-
-This command uses a trained PyLaia model to predict on a dataset.
-
-It requires:
-
-- a [list of image ids](../datasets/index.md#image-names),
-- the pickled `model` file created during [model initialization](../initialization/index.md),
-- the weights `*.ckpt` of the trained model created during [model training](../training/index.md).
-
-## Parameters
-
-The full list of parameters is detailed in this section.
-
-### General parameters
-
-| Parameter | Description | Type | Default |
-| ---------- | ------------------------------------------------------------------------------------------------------------------- | ------ | ------- |
-| `syms` | Positional argument. Path to a file mapping characters to integers. The CTC symbol **must** be mapped to integer 0. | `str` | |
-| `img_list` | Positional argument. File containing the names of the images to decode (one image per line). | `str` | |
-| `img_dirs` | Directories containing line images. | `str` | |
-| `config` | Path to a JSON configuration file | `json` | |
-
-### Common parameters
-
-| Name | Description | Type | Default |
-| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | ------- |
-| `common.train_path` | Directory where the model will be saved | `str` | `.` |
-| `common.model_filename` | Filename of the model. | `str` | `model` |
-| `common.experiment_dirname` | Directory name of the experiment. | `experiment` | `74565` |
-| `common.checkpoint` | Checkpoint to load. Must be a filepath, a filename, a glob pattern or `None` (in this case, the best checkpoint will be loaded). Note that the checkpoint will be searched in `common.experiment_dirname`, unless you provide an absolute filepath. | `int` | `None` |
-
-### Data arguments
-
-| Name | Description | Type | Default |
-| ------------------ | ------------------------------------------------- | ----------- | ------------- |
-| `data.batch_size` | Batch size. | `int` | `8` |
-| `data.color_mode` | Color mode. Must be either `L`, `RGB` or `RGBA`. | `ColorMode` | `ColorMode.L` |
-| `data.num_workers` | Number of worker processes created in dataloaders | `int` | `None` |
-| `data.reading_order` | Reading order on the input lines: LTR (Left-to-Right) or RTL (Right-to-Left). | `ReadingOrder` | `LTR` |
-
-### Decode arguments
-
-| Name | Description | Type | Default |
-| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | --------- |
-| `decode.include_img_ids` | Include the associated image ids in the decoding/segmentation output | `bool` | `True` |
-| `decode.separator` | String to use as a separator between the image ids and the decoding/segmentation output. | `str` | ` ` |
-| `decode.join_string` | String to use to join the decoding output. | `Optional[str]` | ` ` |
-| `decode.use_symbols` | Convert the decoding output to symbols instead of symbol index. | `bool` | `True` |
-| `decode.convert_spaces` | Whether or not to convert spaces. | `bool` | `False` |
-| `decode.input_space` | Replace the space by this symbol if `convert_spaces` is set. Used for word segmentation and confidence score computation. | `str` | `<space>` |
-| `decode.output_space` | Space symbol to display during decoding. | `str` | ` ` |
-| `decode.segmentation` | Use CTC alignment to estimate character or word segmentation. Should be `char` or `word`. | `Optional[str]` | `None ` |
-| `decode.temperature` | Temperature parameters used to scale the logits. | `float` | `1.0` |
-| `decode.print_line_confidence_scores` | Whether to print line confidence scores. | `bool` | `False` |
-| `decode.print_line_confidence_scores` | Whether to print word confidence scores. | `bool` | `False` |
-| `decode.use_language_model` | Whether to decode with an external language model. | `bool` | `False` |
-| `decode.language_model_path` | Path to a KenLM or ARPA n-gram language model. | `str` | `None` |
-| `decode.language_model_weight` | Weight of the language model. | `float` | `None` |
-| `decode.tokens_path` | Path to a file containing valid tokens. If using a file, the expected format is for tokens mapping to the same index to be on the same line. The `ctc` symbol should be at index 0. | `str` | `None` |
-| `decode.lexicon_path` | Path to a lexicon file containing the possible words and corresponding spellings. | `str` | `None` |
-| `decode.unk_token` | String representing unknown characters. | `str` | `<unk>` |
-| `decode.blank_token` | String representing the blank/ctc symbol. | `str` | `<ctc>` |
-
-
-### Logging arguments
-
-| Name | Description | Type | Default |
-| ------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------- |
-| `logging.fmt` | Logging format. | `str` | `%(asctime)s %(levelname)s %(name)s] %(message)s` |
-| `logging.level` | Logging level. Should be in `{NOTSET,DEBUG,INFO,WARNING,ERROR,CRITICAL}` | `Level` | `INFO` |
-| `logging.filepath` | Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname` | `Optional[str]` | |
-| `logging.overwrite` | Whether to overwrite the logfile or to append. | `bool` | `False` |
-| `logging.to_stderr_level` | If filename is set, use this to log also to stderr at the given level. | `Level` | `ERROR` |
-
-### Trainer arguments
-
-Pytorch Lightning `Trainer` flags can also be set using the `--trainer` argument. See [the documentation](https://github.com/Lightning-AI/lightning/blob/1.7.0/docs/source-pytorch/common/trainer.rst#trainer-flags).
-
-This flag is mostly useful to define whether to predict on CPU or GPU.
-
-* `--trainer.gpus 0` to run on CPU,
-* `--trainer.gpus n` to run on `n` GPUs (use with `--training.auto_select True` for auto-selection),
-* `--trainer.gpus -1` to run on all GPUs.
-
-
-## Examples
-
-The prediction can be done using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
-
-We provide some images to try out our models. They can be found in `docs/assets`, on the [Gitlab repository](https://gitlab.teklia.com/atr/pylaia/-/tree/master/docs/assets?ref_type=heads). To test the prediction commands, make sure to download them on your end.
-
-```shell
-mkdir images
-wget https://user-images.githubusercontent.com/100838858/219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f.jpg -P images
-wget https://user-images.githubusercontent.com/100838858/219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4.jpg -P images
-```
-
-### Predict using a model from Hugging Face
-
-First, clone a trained model from Hugging Face:
-```bash
-git clone https://huggingface.co/Teklia/pylaia-huginmunin
-```
-
-!!! note
- Some files are stored through [Git-LFS](https://git-lfs.com/). Make sure all files are correctly pulled using the following command, from the cloned folder.
- ```bash
- git lfs ls-files
- ```
-
- You should see three files:
-
- - the language model (`language_model.arpa.gz`),
- - the model architecture (`model`),
- - the weights (`weights.ckpt`).
-
-List image names in `img_list.txt`:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4
-```
-
-Predict with:
-```bash
-pylaia-htr-decode-ctc --common.experiment_dirname pylaia-huginmunin/ \
- --common.model_filename pylaia-huginmunin/model \
- --img_dir [images] \
- pylaia-huginmunin/syms.txt \
- img_list.txt
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f o g <space> V a l s t a d <space> k a n <space> v i <space> v i s t
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 i k k e <space> g j ø r e <space> R e g n i n g <space> p a a ,
-```
-
-Note that by default, each token is separated by a space, and the space symbol is represented by `--decode.input_space` (default: `"<space>"`).
-
-### Predict with a YAML configuration file
-
-Run the following command to predict a model on CPU using:
-```bash
-pylaia-htr-decode-ctc --config config_decode_model.yaml
-```
-
-With the following configuration file:
-```yaml title="config_decode_model.yaml"
-syms: pylaia-huginmunin/syms.txt
-img_list: img_list.txt
-img_dirs:
- - images/
-common:
- experiment_dirname: pylaia-huginmunin
- model_filename: pylaia-huginmunin/model
-decode:
- join_string: ""
- convert_spaces: true
-trainer:
- gpus: 0
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f og Valstad kan vi vist
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 ikke gjøre Regning paa,
-```
-
-Note that setting `--decode.join_string ""` and `--decode.convert_spaces True` will display the text well formatted.
-
-### Predict with confidence scores
-
-PyLaia estimate character probability for each timestep. It is possible to print the probability at line or word level.
-
-#### Line confidence scores
-
-Run the following command to predict with line confidence scores:
-```bash
-pylaia-htr-decode-ctc --config config_decode_model.yaml \
- --decode.print_line_confidence_score True
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f 0.99 og Valstad kan vi vist
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 0.98 ikke gjøre Regning paa,
-```
-
-#### Word confidence scores
-
-Run the following command to predict with word confidence scores:
-```bash
-pylaia-htr-decode-ctc --config config_decode_model.yaml \
- --decode.print_word_confidence_score True
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f ['1.00', '1.00', '1.00', '1.00', '1.00'] og Valstad kan vi vist
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 ['1.00', '0.91', '1.00', '0.99'] ikke gjøre Regning paa,
-```
-
-#### Temperature scaling
-
-PyLaia tends to output overly confident probabilities. [Temperature scaling](https://arxiv.org/pdf/1706.04599.pdf) can be used to improve the reliability of confidence scores. The best temperature can be determined with a grid search algorithm by maximizing the correlation between 1-CER and confidence scores.
-
-Run the following command to predict callibrated word confidence scores with `temperature=3.0`
-```bash
-pylaia-htr-decode-ctc --config config_decode_model.yaml \
- --decode.print_word_confidence_score True \
- --decode.temperature 3.0
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f ['0.93', '0.85', '0.87', '0.93', '0.85'] og Valstad kan vi vist
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 ['0.93', '0.84', '0.86', '0.83'] ikke gjøre Regning paa,
-```
-
-### Predict with a language model
-
-PyLaia supports KenLM and ARPA language models.
-
-Once the n-gram model is built, run the following command to combine it to your PyLaia model:
-```bash
-pylaia-htr-decode-ctc --config config_decode_model_lm.yaml
-```
-
-With the following configuration file:
-```yaml title="config_decode_model_lm.yaml"
-syms: pylaia-huginmunin/syms.txt
-img_list: img_list.txt
-img_dirs:
- - images/
-common:
- experiment_dirname: pylaia-huginmunin
- model_filename: pylaia-huginmunin/model
-decode:
- join_string: ""
- convert_spaces: true
- use_language_model: true
- language_model_path: pylaia-huginmunin/language_model.arpa.gz
- tokens_path: pylaia-huginmunin/tokens.txt
- lexicon_path: pylaia-huginmunin/lexicon.txt
- language_model_weight: 1.5
- decode.print_line_confidence_score: true
-trainer:
- gpus: 0
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f 0.90 og Valstad kan vi vist
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 0.89 ikke gjøre Regning paa,
-```
-
-### Predict with CTC alignement
-
-It is possible to estimate text localization based on CTC alignments with the `--decode.segmentation` option. It returns a list texts with their estimated coordinates: `(text, x1, y1, x2, y2)`.
-
-#### Character level
-
-To output character localization, use the `--decode.segmentation char` option:
-```bash
-pylaia-htr-decode-ctc --common.experiment_dirname pylaia-huginmunin/ \
- --common.model_filename pylaia-huginmunin/model \
- --decode.segmentation char \
- --img_dir [images] \
- pylaia-huginmunin/syms.txt \
- img_list.txt
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f [('o', 1, 1, 31, 128), ('g', 32, 1, 79, 128), ('<space>', 80, 1, 143, 128), ('V', 144, 1, 167, 128), ('a', 168, 1, 223, 128), ('l', 224, 1, 255, 128), ('s', 256, 1, 279, 128), ('t', 280, 1, 327, 128), ('a', 328, 1, 367, 128), ('d', 368, 1, 407, 128), ('<space>', 408, 1, 496, 128), ('k', 497, 1, 512, 128), ('a', 513, 1, 576, 128), ('n', 577, 1, 624, 128), ('<space>', 625, 1, 712, 128), ('v', 713, 1, 728, 128), ('i', 729, 1, 776, 128), ('<space>', 777, 1, 808, 128), ('v', 809, 1, 824, 128), ('i', 825, 1, 872, 128), ('s', 873, 1, 912, 128), ('t', 913, 1, 944, 128)]
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 [('i', 1, 1, 23, 128), ('k', 24, 1, 71, 128), ('k', 72, 1, 135, 128), ('e', 136, 1, 191, 128), ('<space>', 192, 1, 248, 128), ('g', 249, 1, 264, 128), ('j', 265, 1, 312, 128), ('ø', 313, 1, 336, 128), ('r', 337, 1, 376, 128), ('e', 377, 1, 408, 128), ('<space>', 409, 1, 481, 128), ('R', 482, 1, 497, 128), ('e', 498, 1, 545, 128), ('g', 546, 1, 569, 128), ('n', 570, 1, 601, 128), ('i', 602, 1, 665, 128), ('n', 666, 1, 706, 128), ('g', 707, 1, 762, 128), ('<space>', 763, 1, 794, 128), ('p', 795, 1, 802, 128), ('a', 803, 1, 850, 128), ('a', 851, 1, 890, 128), (',', 891, 1, 914, 128)]
-```
-
-#### Word level
-
-To output word localization, use the `--decode.segmentation word` option:
-```bash
-pylaia-htr-decode-ctc --common.experiment_dirname pylaia-huginmunin/ \
- --common.model_filename pylaia-huginmunin/model \
- --decode.segmentation word \
- --img_dir [images] \
- pylaia-huginmunin/syms.txt \
- img_list.txt
-```
-
-Expected output:
-```text
-219007024-f45433e7-99fd-43b0-bce6-93f63fa72a8f [('og', 1, 1, 79, 128), ('<space>', 80, 1, 143, 128), ('Valstad', 144, 1, 407, 128), ('<space>', 408, 1, 496, 128), ('kan', 497, 1, 624, 128), ('<space>', 625, 1, 712, 128), ('vi', 713, 1, 776, 128), ('<space>', 777, 1, 808, 128), ('vist', 809, 1, 944, 128)]
-219008758-c0097bb4-c55a-4652-ad2e-bba350bee0e4 [('ikke', 1, 1, 191, 128), ('<space>', 192, 1, 248, 128), ('gjøre', 249, 1, 408, 128), ('<space>', 409, 1, 481, 128), ('Regning', 482, 1, 762, 128), ('<space>', 763, 1, 794, 128), ('paa,', 795, 1, 914, 128)]
-```
-
-### Predict on Right-To-Left data
-
-To output word localization, use the `--data.reading_order` option:
-```bash
-pylaia-htr-decode-ctc --common.experiment_dirname pylaia-khatt/ \
- --common.model_filename pylaia-khatt/model \
- --data.reading_order RTL \
- --img_dir [images] \
- pylaia-khatt/syms.txt \
- img_list.txt
-```
-
-Expected output:
-```text
-text_line_1302 العلماء على Ùهم هذه الكتابات بالدراسات اللغوية السامية مثل العبرانية، وباللغة العربية التي
-```
diff --git a/docs/usage/training/index.md b/docs/usage/training/index.md
deleted file mode 100644
index e6858ae188f4e3d5bd2482aa3c62fee905395933..0000000000000000000000000000000000000000
--- a/docs/usage/training/index.md
+++ /dev/null
@@ -1,222 +0,0 @@
-# Training
-
-The `pylaia-htr-train-ctc` command can be used to train a PyLaia model. To know more about the options of this command, use `pylaia-htr-train-ctc --help`.
-
-## Purpose
-
-This command trains a PyLaia architecture on a dataset.
-
-It requires:
-
-- a [formatted dataset](../datasets/index.md),
-- the pickled `model` file created during [model initialization](../initialization/index.md).
-
-!!! note
-
- The [`pylaia-htr-dataset-validate`](../datasets/index.md) command can help you analyze your dataset and point out issues.
-
-## Parameters
-
-The full list of parameters is detailed in this section.
-
-### General parameters
-
-| Parameter | Description | Type | Default |
-| -------------- | ------------------------------------------------------------------------------------------------------------------- | ------ | ------- |
-| `syms` | Positional argument. Path to a file mapping characters to integers. The CTC symbol **must** be mapped to integer 0. | `str` | |
-| `img_dirs` | Positional argument. Directories containing line images. | `str` | |
-| `tr_txt_table` | Positional argument. Path to a file mapping training image ids and tokenized transcription. | `str` | |
-| `va_txt_table` | Positional argument. Path to a file mapping validation image ids and tokenized transcription. | `str` | |
-| `config` | Path to a JSON configuration file | `json` | |
-
-### Common parameters
-
-| Name | Description | Type | Default |
-| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | ---------------- |
-| `common.seed` | Seed for random number generators. | `int` | `74565` |
-| `common.train_path` | Directory where the model will be saved | `str` | `.` |
-| `common.model_filename` | Filename of the model. | `str` | `model` |
-| `common.experiment_dirname` | Directory name of the experiment. | `str` | `experiment` |
-| `common.monitor` | Metric to monitor for early stopping and checkpointing. | `Monitor` | `Monitor.va_cer` |
-| `common.checkpoint` | Checkpoint to load. Must be a filepath, a filename, a glob pattern or `None` (in this case, the best checkpoint will be loaded). Note that the checkpoint will be searched in `common.experiment_dirname`, unless you provide an absolute filepath. | `Optional[str]` | `None` |
-
-### Data arguments
-
-| Name | Description | Type | Default |
-| -------------------- | ----------------------------------------------------------------------------- | -------------- | ------------- |
-| `data.batch_size` | Batch size. | `int` | `8` |
-| `data.color_mode` | Color mode. Must be either `L`, `RGB` or `RGBA`. | `ColorMode` | `ColorMode.L` |
-| `data.num_workers` | Number of worker processes created in dataloaders | `int` | `None` |
-| `data.reading_order` | Reading order on the input lines: LFT (Left-to-Right) or RTL (Right-to-Left). | `ReadingOrder` | `LFT` |
-
-### Train arguments
-
-| Name | Description | Type | Default |
-| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | ------------- |
-| `train.delimiters` | List of symbols representing the word delimiters. | `List` | `["<space>"]` |
-| `train.checkpoint_k` | Model saving mode: `-1` all models will be saved, `0`: no models are saved, `k` the `k` best models are saved. | `int` | `3` |
-| `train.resume` | Whether to resume training with a checkpoint. This option can be used to continue training on the same dataset. | `bool` | `False` |
-| `train.pretrain` | Whether to load pretrained weights from a checkpoint. This option can be used to load pretrained weights when fine-tuning a model on a new dataset. | `bool` | `False` |
-| `train.freeze_layers` | List of layers to freeze during training: `"conv"` to freeze convolutional layers, `"rnn"` to freeze recurrent layers, `"linear"` to freeze the linear layer | `List[str]` | `None` |
-| `train.early_stopping_patience` | Number of validation epochs with no improvement after which training will be stopped. | `int` | `20` |
-| `train.gpu_stats` | Whether to include GPU stats in the training progress bar. | `bool` | `False` |
-| `train.augment_training` | Whether to use data augmentation. | `bool` | `False` |
-| `train.log_to_wandb` | Whether to log training metrics and parameters to Weights & Biases. | `bool` | `False` |
-
-
-### Logging arguments
-
-| Name | Description | Type | Default |
-| ------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------- |
-| `logging.fmt` | Logging format. | `str` | `%(asctime)s %(levelname)s %(name)s] %(message)s` |
-| `logging.level` | Logging level. Should be in `{NOTSET,DEBUG,INFO,WARNING,ERROR,CRITICAL}` | `Level` | `INFO` |
-| `logging.filepath` | Filepath for the logs file. Can be a filepath or a filename to be created in `train_path`/`experiment_dirname` | `Optional[str]` | |
-| `logging.overwrite` | Whether to overwrite the logfile or to append. | `bool` | `False` |
-| `logging.to_stderr_level` | If filename is set, use this to log also to stderr at the given level. | `Level` | `ERROR` |
-
-### Optimizer arguments
-
-| Name | Description | Type | Default |
-| ------------------------------ | --------------------------------------------------------- | ------- | --------- |
-| `optimizers.name` | Optimization algorithm. Must be `SGD`, `RMSProp`, `Adam`. | `List` | `RMSProp` |
-| `optimizers.learning_rate` | Learning rate. | `float` | `0.0005` |
-| `optimizers.momentum` | Momentum. | `float` | `0.0` |
-| `optimizers.weight_l2_penalty` | Apply this L2 weight penalty to the loss function. | `float` | `0.0` |
-| `optimizers.nesterov` | Whether to use Nesterov momentum. | `bool` | `False` |
-
-### Scheduler arguments
-
-| Name | Description | Type | Default |
-| -------------------- | ------------------------------------------------------------------------------- | --------- | ----------------- |
-| `scheduler.active` | Whether to use an on-plateau learning rate scheduler. | `bool` | `False` |
-| `scheduler.monitor` | Metric for the scheduler to monitor. | `Monitor` | `Monitor.va_loss` |
-| `scheduler.patience` | Number of epochs with no improvement after which learning rate will be reduced. | `int` | `5` |
-| `scheduler.factor` | Factor by which the learning rate will be reduced. | `float` | `0.1` |
-
-### Trainer arguments
-
-Pytorch Lighning `Trainer` flags can also be set using the `--trainer` argument. See [the documentation](https://github.com/Lightning-AI/lightning/blob/1.7.0/docs/source-pytorch/common/trainer.rst#trainer-flags).
-
-
-## Examples
-
-The model can be trained using command-line arguments or a YAML configuration file. Note that CLI arguments override the values from the configuration file.
-
-
-### Train from scratch with Command Line Arguments (CLI)
-
-Run the following command to train a model:
-```sh
-pylaia-htr-train-ctc /path/to/syms.txt \
- `cat img_dirs_args.txt`\
- /path/to/train.txt \
- /path/to/val.txt \
- --trainer.gpus 1 \
- --data.batch_size 32
-```
-
-### Train from scratch with a YAML configuration file
-
-Run the following command to train a model:
-```sh
-pylaia-htr-train-ctc --config config_train_model.yaml
-```
-
-Where `config_train_model.yaml` is:
-
-```yaml
-syms: /path/to/syms.txt
-img_dirs:
- - /path/to/images/
-tr_txt_table: /path/to/train.txt
-va_txt_table: /path/to/val.txt
-common:
- experiment_dirname: experiment-dataset
-logging:
- filepath: pylaia_training.log
-scheduler:
- active: true
-train:
- augment_training: true
- early_stopping_patience: 80
-trainer:
- auto_select_gpus: true
- gpus: 1
- max_epochs: 600
-```
-
-### Resume training from a checkpoint
-
-Run the following command to continue training from a checkpoint for 200 epochs.
-```sh
-pylaia-htr-train-ctc --config config_train_model.yaml --train.resume true --trainer.max_epochs 200
-```
-
-!!! note
- If `common.checkpoint` is not set, PyLaia will select the best checkpoint from `common.experiment_dirname`
-
-### Fine-tune from a checkpoint
-
-Run the following command to load pretrained weights and fine-tune on a new dataset for 200 epochs.
-```sh
-pylaia-htr-train-ctc --config config_train_model.yaml --common.experiment_dirname experiment/ --common.checkpoint initial_checkpoint.ckpt --train.pretrain true --trainer.max_epochs 200
-```
-
-!!! warning
- This option requires that your model architecture `model` matches the one used to train `initial_checkpoint.ckpt`.
- The last linear layer will be reinitialized using the Xavier initialization to match the new vocabulary size.
-
-!!! note
- The initial checkpoint is expected to be in the following directory: `{common.experiment_dirname}/pretrained/`.
- If it is located in `common.experiment_dirname`, the subdirectory `pretrained` will be created and the checkpoint will be moved there automatically.
-
-### Train on Right-To-Left reading order
-
-By default, PyLaia expects images with Left-to-Right reading order.
-To train a model on Right-To-Left data, use the following command:
-```sh
-pylaia-htr-train-ctc --config config_train_model_rtl.yaml
-```
-
-Where `config_train_model_rtl.yaml` is:
-
-```yaml title="config_train_model_rtl.yaml"
-syms: /path/to/syms.txt
-img_dirs:
- - /path/to/images/
-tr_txt_table: /path/to/train.txt
-va_txt_table: /path/to/val.txt
-common:
- experiment_dirname: experiment-dataset
-logging:
- filepath: pylaia_training.log
-scheduler:
- active: true
-train:
- augment_training: true
- early_stopping_patience: 80
-trainer:
- auto_select_gpus: true
- gpus: 1
- max_epochs: 600
-data:
- reading_order: RTL
-```
-
-### Train and log to Weights & Biases
-
-By default, PyLaia logs metrics and losses to a local CSV file. You can chose to log into [Weights & Biases](https://wandb.ai/home) instead.
-
-To set up Weights & Biases:
-* Run `pip install pylaia[wandb]` to install the required dependencies
-* Sign in to Weights & Biases using `wandb login`
-
-Then, start training with `pylaia-htr-train-ctc --config config_train_model.yaml --train.log_to_wandb true`.
-
-This will create a project called `PyLaia` in W&B with one run for each training. The following are monitored for each run:
-* Training and validation metrics (losses, CER, WER)
-* Model gradients
-* System metrics (GPU and CPU utilisation, temperature, allocated memory)
-* Hyperparameters (training configuration)
-
-A public dashboard is available [here](https://wandb.ai/starride-teklia/PyLaia%20demo) as an example.
diff --git a/mkdocs.yml b/mkdocs.yml
deleted file mode 100644
index da08526cc7a834da8c05e6e58af9eb1c0dbf3858..0000000000000000000000000000000000000000
--- a/mkdocs.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-site_name: PyLaia
-site_dir: public
-
-theme:
- name: material
-
- font:
- text: Roboto
- code: Roboto Mono
-
- features:
- - navigation.top
- - navigation.tracking
- - navigation.indexes
- - navigation.instant
- - navigation.instant.progress
- - content.code.copy
-
- palette:
- # Palette toggle for light mode
- - media: "(prefers-color-scheme: light)"
- scheme: default
- toggle:
- icon: material/brightness-7
- name: Switch to dark mode
-
- # Palette toggle for dark mode
- - media: "(prefers-color-scheme: dark)"
- scheme: slate
- toggle:
- icon: material/brightness-4
- name: Switch to light mode
-
-plugins:
-- search
-- autorefs
-- literate-nav:
- nav_file: SUMMARY.md
-- mkdocstrings:
- handlers:
- python:
- options:
- show_root_toc_entry: false
- show_object_full_path: false
- show_root_heading: yes
- show_source: true
- docstring_style: google
- merge_init_into_class: yes
- show_category_heading: yes
- show_signature_annotations: yes
- separate_signature: yes
- members_order: source
- unwrap_annotated: yes
- show_if_no_docstring: yes
- filters:
- - "!^_"
- - "!^__"
-
-nav:
- - Home: index.md
- - Original paper: original_paper.md
- - Get started:
- - get_started/index.md
- - Development: get_started/development.md
- - Usage:
- - usage/index.md
- - Dataset:
- - usage/datasets/index.md
- - Dataset formatting: usage/datasets/format.md
- - Model initialization: usage/initialization/index.md
- - Training: usage/training/index.md
- - Prediction: usage/prediction/index.md
- - Netout: usage/netout/index.md
- - Explicit language modeling: usage/language_models/index.md
- # defer to literate-nav
- - Code Reference: reference/
- - Releases: releases.md
-
-copyright: Copyright © Teklia
-
-extra:
- social:
- - icon: fontawesome/brands/gitlab
- name: Git repository for this project
- link: https://gitlab.teklia.com/atr/pylaia
-
-markdown_extensions:
- - pymdownx.superfences
- - admonition
diff --git a/package.json b/package.json
new file mode 100644
index 0000000000000000000000000000000000000000..5bc5ecabc28a78dab8e594b037f89793b7783861
--- /dev/null
+++ b/package.json
@@ -0,0 +1,6 @@
+{
+ "dependencies": {
+ "@antora/lunr-extension": "^1.0.0-alpha.9",
+ "antora": "^3.1.10"
+ }
+}
diff --git a/pyproject.toml b/pyproject.toml
index 79672355c445c332e5375503b73dc0373776eec2..3d8dce8a57273053f47daecde2c2458e4ede6e7f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,15 +56,6 @@ dev = [
test = [
"tox==4.18.0",
]
-docs = [
- "black==24.8.0",
- "mkdocs-autorefs==1.1.0",
- "mkdocs-gen-files==0.5.0",
- "mkdocs-literate-nav==0.6.1",
- "mkdocs-material==9.5.33",
- "mkdocs-section-index==0.3.9",
- "mkdocstrings-python==1.10.8",
-]
wandb = ["wandb==0.18.5"]
[project.scripts]