diff --git a/README.md b/README.md index f20bca8c90e845525ad1cc7d02cef390b9a651af..2a3848e0020239c8cdd0f1e81fcc90fd91b7c5b8 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ $ nerval -a demo/toy_test_annot.bio -p demo/toy_test_predict.bio You can also indicate a folder and a csv file to have multiple evaluation at once. ``` -$ nerval -a demo/annot/ -p demo/pred/ -c demo/cor.csv +$ nerval -c demo/mapping_file.csv -f demo/bio_folder ``` ## Metric diff --git a/demo/cor.csv b/demo/mapping_file.csv similarity index 83% rename from demo/cor.csv rename to demo/mapping_file.csv index a0f41c651ad7e208a03a7db632292053910d4e69..5a2ce9244d8751ec5812a297c8b7ddf367ed3a56 100644 --- a/demo/cor.csv +++ b/demo/mapping_file.csv @@ -1,3 +1,2 @@ -annot,predict demo_annot.bio,demo_predict.bio toy_test_annot.bio,toy_test_predict.bio \ No newline at end of file diff --git a/nerval/evaluate.py b/nerval/evaluate.py index ebdba10c642e161eed19e7256be6547c16cbfd4b..7fa887f28221441280738d4c17e9f425a4610585 100644 --- a/nerval/evaluate.py +++ b/nerval/evaluate.py @@ -5,11 +5,11 @@ import glob import logging import os import re +from csv import reader from pathlib import Path import editdistance import edlib -import pandas as pd import termtables as tt NOT_ENTITY_TAG = "O" @@ -542,13 +542,15 @@ def run(annotation: str, prediction: str, threshold: int) -> dict: def run_multiple(file_csv, folder, threshold): """Run the program for multiple files (correlation indicated in the csv file)""" - # Read the csv in a dataframe - df_cor = pd.read_csv(file_csv) + # Read the csv in a list + with open(file_csv, "r") as read_obj: + csv_reader = reader(read_obj) + list_cor = list(csv_reader) if os.path.isdir(folder): list_bio_file = glob.glob(str(folder) + "/**/*.bio", recursive=True) - for index, row in df_cor.iterrows(): + for row in list_cor: annot = None predict = None @@ -560,11 +562,11 @@ def run_multiple(file_csv, folder, threshold): predict = file if annot and predict: + print(os.path.basename(predict)) run(annot, predict, threshold) print() else: - raise f"No file found for row {index}" - + raise f"No file found for files {annot}, {predict}" else: raise "This is no folder" diff --git a/requirements.txt b/requirements.txt index 7ae8377e3effb0c3297432bc96f28dff0db8555b..d6af2d0799ba16c43c624bc269a5cc78a21c9758 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ editdistance==0.5.3 edlib==1.3.8.post2 -pandas==1.3.4 termtables==0.2.3 diff --git a/tests/test_folder/test_demo_annot.bio b/tests/test_folder/test_demo_annot.bio new file mode 100644 index 0000000000000000000000000000000000000000..cf16200fcd0436d943f71127f63894e428fafea7 --- /dev/null +++ b/tests/test_folder/test_demo_annot.bio @@ -0,0 +1,82 @@ +Césaire B-PER +Alphonse I-PER +Garon I-PER +marraine O +Adeline B-PER +Dionne I-PER +, O +soussignés O +Lecture O +faite O +Adéline O +Dionne O +Arsène O +Côté O +Arpin O +R O +Le O +onze B-DAT +aout I-DAT +mil I-DAT +neuf I-DAT +cent I-DAT +un I-DAT +nous O +prêtre O +soussigné O +avons O +baptisé O +Marie B-PER +Luce I-PER +Louise I-PER +, O +née O +la B-DAT +veille I-DAT +, O +fille O +légitime O +de O +Carmel B-PER +Côté I-PER +, O +cordonnier B-OCC +, O +pré O +- O +sent O +, O +déclarant O +ne O +savoir O +signer O +, O +et O +de O +Eugé B-PER +nie I-PER +Fréchette I-PER +, O +de O +cette B-LOC +paroisse I-LOC +. O +Parrain O +Napoléon B-PER +Fréchette I-PER +, O +marraine O +Adeline B-PER +Tremblay I-PER +, O +soussignés O +, O +de O +Ste B-LOC +Luce I-LOC +, O +Lec O +- O +ture O +faite O +. O diff --git a/tests/test_folder/test_demo_predict.bio b/tests/test_folder/test_demo_predict.bio new file mode 100644 index 0000000000000000000000000000000000000000..7e01c2d127aa47c95c49ab87a06e06d86f9af9b0 --- /dev/null +++ b/tests/test_folder/test_demo_predict.bio @@ -0,0 +1,81 @@ +Césaire B-PER +Alphonse O +Garon B-PER +marraine O +Adeline B-PER +Dionne I-PER +, O +soussignés O +Lecture O +faite O +Adéline O +Dionne O +Arsène O +Côté O +Arpin O +R O +Le O +onze B-DAT +aout I-DAT +mil I-DAT +neuf I-DAT +cent I-DAT +un O +nous O +pretre O +soussigné O +avons O +baptisé O +Marie B-PER +Luce I-PER +Louise I-PER +, O +née O +la B-DAT +veille I-DAT +, O +fille O +légitime O +de O +Carmel B-PER +Côté I-PER +, O +cordonnier B-OCC +, O +pré O +- O +sent O +, O +déclarant O +ne O +savoir O +signer O +, O +et O +de O +Eugé B-PER +nie I-PER +Fréchette I-PER +, O +de O +cette B-LOC +paroisse I-LOC +. O +Parrain O +Napoléon B-PER +Fréchette I-PER +, O +marraine O +Adéline B-PER +Tremblay I-PER +, O +sousignés O +, O +de O +St B-LOC +. I-LOC +Luce O +, O +Lec O +ture O +faite O diff --git a/tests/test_folder/test_toy_annot.bio b/tests/test_folder/test_toy_annot.bio new file mode 100644 index 0000000000000000000000000000000000000000..5a941ee8ce361e67dbd213b2b2cd5d6a6a53e4ef --- /dev/null +++ b/tests/test_folder/test_toy_annot.bio @@ -0,0 +1,39 @@ +John B-PER +Ronald I-PER +Reuel I-PER +Tolkien I-PER +was O +born O +on O +three B-DAT +January I-DAT +eighteen I-DAT +ninety I-DAT +- I-DAT +two I-DAT +in O +Bloemfontein B-LOC +in O +the O +Orange B-LOC +Free I-LOC +State I-LOC +, O +to O +Arthur B-PER +Reuel I-PER +Tolkien I-PER +, O +an O +English O +bank B-OCC +manager I-OCC +, O +and O +his O +wife O +Mabel B-PER +, O +née O +Suffield B-PER +. O diff --git a/tests/test_folder/test_toy_predict.bio b/tests/test_folder/test_toy_predict.bio new file mode 100644 index 0000000000000000000000000000000000000000..5a941ee8ce361e67dbd213b2b2cd5d6a6a53e4ef --- /dev/null +++ b/tests/test_folder/test_toy_predict.bio @@ -0,0 +1,39 @@ +John B-PER +Ronald I-PER +Reuel I-PER +Tolkien I-PER +was O +born O +on O +three B-DAT +January I-DAT +eighteen I-DAT +ninety I-DAT +- I-DAT +two I-DAT +in O +Bloemfontein B-LOC +in O +the O +Orange B-LOC +Free I-LOC +State I-LOC +, O +to O +Arthur B-PER +Reuel I-PER +Tolkien I-PER +, O +an O +English O +bank B-OCC +manager I-OCC +, O +and O +his O +wife O +Mabel B-PER +, O +née O +Suffield B-PER +. O diff --git a/tests/test_mapping_file.csv b/tests/test_mapping_file.csv new file mode 100644 index 0000000000000000000000000000000000000000..5a2ce9244d8751ec5812a297c8b7ddf367ed3a56 --- /dev/null +++ b/tests/test_mapping_file.csv @@ -0,0 +1,2 @@ +demo_annot.bio,demo_predict.bio +toy_test_annot.bio,toy_test_predict.bio \ No newline at end of file diff --git a/tests/test_run.py b/tests/test_run.py index d0ea6d59880e5da982a949b80818215af5358160..cedbd0d169f8336763927606f744569ef940f1d3 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -9,6 +9,8 @@ FAKE_ANNOT_BIO = "tests/test_annot.bio" FAKE_PREDICT_BIO = "tests/test_predict.bio" EMPTY_BIO = "tests/test_empty.bio" FAKE_BIO_NESTED = "tests/test_nested.bio" +BIO_FOLDER = "test_folder" +CSV_FILE = "test_mapping_file.csv" expected_scores_nested = { "All": { @@ -81,3 +83,8 @@ def test_run_empty_bio(): def test_run_empty_entry(): with pytest.raises(TypeError): evaluate.run(None, None, THRESHOLD) + + +def test_run_multiple(): + with pytest.raises(Exception): + evaluate.run_multiple(CSV_FILE, BIO_FOLDER, THRESHOLD)