diff --git a/README.md b/README.md index 637e0c80cf00311e2353b0ba4c390e4f7b9fbd08..2a3848e0020239c8cdd0f1e81fcc90fd91b7c5b8 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Nerval is an evaluation library written in python implementing a metric for named-entity recognition evaluation on noisy text, typically to measure NER performances on OCR or Handwritten text recognition predictions. Expected inputs are a ground truth and a prediction BIOES/BILOU files without any ''§'' occurrences, this character having a special meaning during evaluation. +It also works by designating a csv file with file matches (one pair per row with the annotation file in the first column and the prediction file in the second column) ## Usage @@ -31,7 +32,7 @@ You can now use Nerval in command line : ``` $ nerval -a/--annot <annot_file.bio> -p/--predict <predict-file.bio> \ - [-t/--threshold <threshold_value>] + [-t/--threshold <threshold_value>] [-c/--csv <correspondence_file.csv>] ``` The threshold value should be between 0 and 1. It designates the acceptable number of characters differing between an annotated and a predicted entity - over the number of characters in the annotated entity - to consider it as a match. Default value is 0.30. 0 would impose perfect matches, 1 would allow completely different strings to be considered as a match. @@ -58,6 +59,12 @@ We also provide two annotation and prediction toy files, which are identical for $ nerval -a demo/toy_test_annot.bio -p demo/toy_test_predict.bio ``` +You can also indicate a folder and a csv file to have multiple evaluation at once. + +``` +$ nerval -c demo/mapping_file.csv -f demo/bio_folder +``` + ## Metric This metric uses string alignment at character level. diff --git a/demo/bio_folder/demo_annot.bio b/demo/bio_folder/demo_annot.bio new file mode 100644 index 0000000000000000000000000000000000000000..cf16200fcd0436d943f71127f63894e428fafea7 --- /dev/null +++ b/demo/bio_folder/demo_annot.bio @@ -0,0 +1,82 @@ +Césaire B-PER +Alphonse I-PER +Garon I-PER +marraine O +Adeline B-PER +Dionne I-PER +, O +soussignés O +Lecture O +faite O +Adéline O +Dionne O +Arsène O +Côté O +Arpin O +R O +Le O +onze B-DAT +aout I-DAT +mil I-DAT +neuf I-DAT +cent I-DAT +un I-DAT +nous O +prêtre O +soussigné O +avons O +baptisé O +Marie B-PER +Luce I-PER +Louise I-PER +, O +née O +la B-DAT +veille I-DAT +, O +fille O +légitime O +de O +Carmel B-PER +Côté I-PER +, O +cordonnier B-OCC +, O +pré O +- O +sent O +, O +déclarant O +ne O +savoir O +signer O +, O +et O +de O +Eugé B-PER +nie I-PER +Fréchette I-PER +, O +de O +cette B-LOC +paroisse I-LOC +. O +Parrain O +Napoléon B-PER +Fréchette I-PER +, O +marraine O +Adeline B-PER +Tremblay I-PER +, O +soussignés O +, O +de O +Ste B-LOC +Luce I-LOC +, O +Lec O +- O +ture O +faite O +. O diff --git a/demo/bio_folder/demo_predict.bio b/demo/bio_folder/demo_predict.bio new file mode 100644 index 0000000000000000000000000000000000000000..7e01c2d127aa47c95c49ab87a06e06d86f9af9b0 --- /dev/null +++ b/demo/bio_folder/demo_predict.bio @@ -0,0 +1,81 @@ +Césaire B-PER +Alphonse O +Garon B-PER +marraine O +Adeline B-PER +Dionne I-PER +, O +soussignés O +Lecture O +faite O +Adéline O +Dionne O +Arsène O +Côté O +Arpin O +R O +Le O +onze B-DAT +aout I-DAT +mil I-DAT +neuf I-DAT +cent I-DAT +un O +nous O +pretre O +soussigné O +avons O +baptisé O +Marie B-PER +Luce I-PER +Louise I-PER +, O +née O +la B-DAT +veille I-DAT +, O +fille O +légitime O +de O +Carmel B-PER +Côté I-PER +, O +cordonnier B-OCC +, O +pré O +- O +sent O +, O +déclarant O +ne O +savoir O +signer O +, O +et O +de O +Eugé B-PER +nie I-PER +Fréchette I-PER +, O +de O +cette B-LOC +paroisse I-LOC +. O +Parrain O +Napoléon B-PER +Fréchette I-PER +, O +marraine O +Adéline B-PER +Tremblay I-PER +, O +sousignés O +, O +de O +St B-LOC +. I-LOC +Luce O +, O +Lec O +ture O +faite O diff --git a/demo/bio_folder/toy_test_annot.bio b/demo/bio_folder/toy_test_annot.bio new file mode 100644 index 0000000000000000000000000000000000000000..5a941ee8ce361e67dbd213b2b2cd5d6a6a53e4ef --- /dev/null +++ b/demo/bio_folder/toy_test_annot.bio @@ -0,0 +1,39 @@ +John B-PER +Ronald I-PER +Reuel I-PER +Tolkien I-PER +was O +born O +on O +three B-DAT +January I-DAT +eighteen I-DAT +ninety I-DAT +- I-DAT +two I-DAT +in O +Bloemfontein B-LOC +in O +the O +Orange B-LOC +Free I-LOC +State I-LOC +, O +to O +Arthur B-PER +Reuel I-PER +Tolkien I-PER +, O +an O +English O +bank B-OCC +manager I-OCC +, O +and O +his O +wife O +Mabel B-PER +, O +née O +Suffield B-PER +. O diff --git a/demo/bio_folder/toy_test_predict.bio b/demo/bio_folder/toy_test_predict.bio new file mode 100644 index 0000000000000000000000000000000000000000..5a941ee8ce361e67dbd213b2b2cd5d6a6a53e4ef --- /dev/null +++ b/demo/bio_folder/toy_test_predict.bio @@ -0,0 +1,39 @@ +John B-PER +Ronald I-PER +Reuel I-PER +Tolkien I-PER +was O +born O +on O +three B-DAT +January I-DAT +eighteen I-DAT +ninety I-DAT +- I-DAT +two I-DAT +in O +Bloemfontein B-LOC +in O +the O +Orange B-LOC +Free I-LOC +State I-LOC +, O +to O +Arthur B-PER +Reuel I-PER +Tolkien I-PER +, O +an O +English O +bank B-OCC +manager I-OCC +, O +and O +his O +wife O +Mabel B-PER +, O +née O +Suffield B-PER +. O diff --git a/demo/mapping_file.csv b/demo/mapping_file.csv new file mode 100644 index 0000000000000000000000000000000000000000..5a2ce9244d8751ec5812a297c8b7ddf367ed3a56 --- /dev/null +++ b/demo/mapping_file.csv @@ -0,0 +1,2 @@ +demo_annot.bio,demo_predict.bio +toy_test_annot.bio,toy_test_predict.bio \ No newline at end of file diff --git a/tests/test_folder/test_demo_annot.bio b/tests/test_folder/test_demo_annot.bio new file mode 100644 index 0000000000000000000000000000000000000000..cf16200fcd0436d943f71127f63894e428fafea7 --- /dev/null +++ b/tests/test_folder/test_demo_annot.bio @@ -0,0 +1,82 @@ +Césaire B-PER +Alphonse I-PER +Garon I-PER +marraine O +Adeline B-PER +Dionne I-PER +, O +soussignés O +Lecture O +faite O +Adéline O +Dionne O +Arsène O +Côté O +Arpin O +R O +Le O +onze B-DAT +aout I-DAT +mil I-DAT +neuf I-DAT +cent I-DAT +un I-DAT +nous O +prêtre O +soussigné O +avons O +baptisé O +Marie B-PER +Luce I-PER +Louise I-PER +, O +née O +la B-DAT +veille I-DAT +, O +fille O +légitime O +de O +Carmel B-PER +Côté I-PER +, O +cordonnier B-OCC +, O +pré O +- O +sent O +, O +déclarant O +ne O +savoir O +signer O +, O +et O +de O +Eugé B-PER +nie I-PER +Fréchette I-PER +, O +de O +cette B-LOC +paroisse I-LOC +. O +Parrain O +Napoléon B-PER +Fréchette I-PER +, O +marraine O +Adeline B-PER +Tremblay I-PER +, O +soussignés O +, O +de O +Ste B-LOC +Luce I-LOC +, O +Lec O +- O +ture O +faite O +. O diff --git a/tests/test_folder/test_demo_predict.bio b/tests/test_folder/test_demo_predict.bio new file mode 100644 index 0000000000000000000000000000000000000000..7e01c2d127aa47c95c49ab87a06e06d86f9af9b0 --- /dev/null +++ b/tests/test_folder/test_demo_predict.bio @@ -0,0 +1,81 @@ +Césaire B-PER +Alphonse O +Garon B-PER +marraine O +Adeline B-PER +Dionne I-PER +, O +soussignés O +Lecture O +faite O +Adéline O +Dionne O +Arsène O +Côté O +Arpin O +R O +Le O +onze B-DAT +aout I-DAT +mil I-DAT +neuf I-DAT +cent I-DAT +un O +nous O +pretre O +soussigné O +avons O +baptisé O +Marie B-PER +Luce I-PER +Louise I-PER +, O +née O +la B-DAT +veille I-DAT +, O +fille O +légitime O +de O +Carmel B-PER +Côté I-PER +, O +cordonnier B-OCC +, O +pré O +- O +sent O +, O +déclarant O +ne O +savoir O +signer O +, O +et O +de O +Eugé B-PER +nie I-PER +Fréchette I-PER +, O +de O +cette B-LOC +paroisse I-LOC +. O +Parrain O +Napoléon B-PER +Fréchette I-PER +, O +marraine O +Adéline B-PER +Tremblay I-PER +, O +sousignés O +, O +de O +St B-LOC +. I-LOC +Luce O +, O +Lec O +ture O +faite O diff --git a/tests/test_folder/test_toy_annot.bio b/tests/test_folder/test_toy_annot.bio new file mode 100644 index 0000000000000000000000000000000000000000..5a941ee8ce361e67dbd213b2b2cd5d6a6a53e4ef --- /dev/null +++ b/tests/test_folder/test_toy_annot.bio @@ -0,0 +1,39 @@ +John B-PER +Ronald I-PER +Reuel I-PER +Tolkien I-PER +was O +born O +on O +three B-DAT +January I-DAT +eighteen I-DAT +ninety I-DAT +- I-DAT +two I-DAT +in O +Bloemfontein B-LOC +in O +the O +Orange B-LOC +Free I-LOC +State I-LOC +, O +to O +Arthur B-PER +Reuel I-PER +Tolkien I-PER +, O +an O +English O +bank B-OCC +manager I-OCC +, O +and O +his O +wife O +Mabel B-PER +, O +née O +Suffield B-PER +. O diff --git a/tests/test_folder/test_toy_predict.bio b/tests/test_folder/test_toy_predict.bio new file mode 100644 index 0000000000000000000000000000000000000000..5a941ee8ce361e67dbd213b2b2cd5d6a6a53e4ef --- /dev/null +++ b/tests/test_folder/test_toy_predict.bio @@ -0,0 +1,39 @@ +John B-PER +Ronald I-PER +Reuel I-PER +Tolkien I-PER +was O +born O +on O +three B-DAT +January I-DAT +eighteen I-DAT +ninety I-DAT +- I-DAT +two I-DAT +in O +Bloemfontein B-LOC +in O +the O +Orange B-LOC +Free I-LOC +State I-LOC +, O +to O +Arthur B-PER +Reuel I-PER +Tolkien I-PER +, O +an O +English O +bank B-OCC +manager I-OCC +, O +and O +his O +wife O +Mabel B-PER +, O +née O +Suffield B-PER +. O diff --git a/tests/test_mapping_file.csv b/tests/test_mapping_file.csv new file mode 100644 index 0000000000000000000000000000000000000000..5a2ce9244d8751ec5812a297c8b7ddf367ed3a56 --- /dev/null +++ b/tests/test_mapping_file.csv @@ -0,0 +1,2 @@ +demo_annot.bio,demo_predict.bio +toy_test_annot.bio,toy_test_predict.bio \ No newline at end of file diff --git a/tests/test_run.py b/tests/test_run.py index 8505b15bb9ba12467282c72eef6c31e112d25545..4a6e9d598c758b3eb26e538f62407b5601747256 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -9,6 +9,8 @@ FAKE_ANNOT_BIO = "tests/test_annot.bio" FAKE_PREDICT_BIO = "tests/test_predict.bio" EMPTY_BIO = "tests/test_empty.bio" FAKE_BIO_NESTED = "tests/test_nested.bio" +BIO_FOLDER = "test_folder" +CSV_FILE = "test_mapping_file.csv" expected_scores_nested = { "All": { @@ -81,3 +83,8 @@ def test_run_empty_bio(): def test_run_empty_entry(): with pytest.raises(TypeError): evaluate.run(None, None, THRESHOLD) + + +def test_run_multiple(): + with pytest.raises(Exception): + evaluate.run_multiple(CSV_FILE, BIO_FOLDER, THRESHOLD)