diff --git a/environment.yml b/environment.yml index b62ee1eda5c5a9a7b8fbbd0ef1d9ddf496e1a45d..66ebd3081f94d2f40b527c90169c477fc639a11a 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: discret +name: testenv channels: - defaults dependencies: @@ -23,10 +23,17 @@ dependencies: - zlib=1.2.13=h5eee18b_0 - pip: - adapter-transformers==3.2.1 + - aiohttp==3.8.4 + - aiosignal==1.3.1 + - async-timeout==4.0.2 + - attrs==23.1.0 - certifi==2023.5.7 - charset-normalizer==3.1.0 - cmake==3.26.3 + - datasets==2.12.0 + - dill==0.3.6 - filelock==3.12.0 + - frozenlist==1.3.3 - fsspec==2023.5.0 - huggingface-hub==0.14.1 - idna==3.4 @@ -35,6 +42,8 @@ dependencies: - lit==16.0.3 - markupsafe==2.1.2 - mpmath==1.3.0 + - multidict==6.0.4 + - multiprocess==0.70.14 - networkx==3.1 - numpy==1.24.3 - nvidia-cublas-cu11==11.10.3.66 @@ -51,11 +60,13 @@ dependencies: - packaging==23.1 - pandas==2.0.1 - pillow==9.5.0 + - pyarrow==12.0.0 - python-dateutil==2.8.2 - pytz==2023.3 - pyyaml==6.0 - regex==2023.5.5 - requests==2.30.0 + - responses==0.18.0 - scikit-learn==1.2.2 - scipy==1.10.1 - six==1.16.0 @@ -70,4 +81,7 @@ dependencies: - triton==2.0.0 - typing-extensions==4.5.0 - tzdata==2023.3 - - urllib3==2.0.2 \ No newline at end of file + - urllib3==2.0.2 + - xxhash==3.2.0 + - yarl==1.9.2 +prefix: /home/lena/anaconda3/envs/testenv diff --git a/requirements.txt b/requirements.txt index 95634697cad66af4762c7c2bfaa4936e55dc8871..b24b999bbe0223dc88858691081d380f42c4b0b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,15 @@ adapter-transformers==3.2.1 +aiohttp==3.8.4 +aiosignal==1.3.1 +async-timeout==4.0.2 +attrs==23.1.0 certifi==2023.5.7 charset-normalizer==3.1.0 cmake==3.26.3 +datasets==2.12.0 +dill==0.3.6 filelock==3.12.0 +frozenlist==1.3.3 fsspec==2023.5.0 huggingface-hub==0.14.1 idna==3.4 @@ -11,6 +18,8 @@ joblib==1.2.0 lit==16.0.3 MarkupSafe==2.1.2 mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.14 networkx==3.1 numpy==1.24.3 nvidia-cublas-cu11==11.10.3.66 @@ -27,11 +36,13 @@ nvidia-nvtx-cu11==11.7.91 packaging==23.1 pandas==2.0.1 Pillow==9.5.0 +pyarrow==12.0.0 python-dateutil==2.8.2 pytz==2023.3 PyYAML==6.0 regex==2023.5.5 requests==2.30.0 +responses==0.18.0 scikit-learn==1.2.2 scipy==1.10.1 six==1.16.0 @@ -46,4 +57,6 @@ transformers==4.29.0 triton==2.0.0 typing_extensions==4.5.0 tzdata==2023.3 -urllib3==2.0.2 \ No newline at end of file +urllib3==2.0.2 +xxhash==3.2.0 +yarl==1.9.2 diff --git a/train_classifiers.sh b/train_classifiers.sh index 538257c56839a31f0b0182821d2cbcfac0b57d99..7ba5e72a085db190487e7fe57fa167075e9652bb 100644 --- a/train_classifiers.sh +++ b/train_classifiers.sh @@ -1,12 +1,22 @@ #!/usr/bin/env bash +# IMPORTANT: Add the path to your cloned DISRPT repo! +# This script will train all three classifiers (and 2 adapters) +# needed for this task. +# First script does not need the adapters, but the last 2 ones do. +# If you do not have the adapters, please run all five scripts. +# If you have the adapters trained, +# you can directly run the classifiers. + +# To reproduce the results of MELODI team, do NOT change other params. + # bare classifier -python pytorch_classifier.py --num_epochs 6 --data_path '../sharedtask2023/data' +python pytorch_classifier.py --num_epochs 6 --data_path 'PATH_TO_REPO' # Train the adapters: -python make_adapter.py --num_epochs 15 --data_path '../sharedtask2023/data' --freeze_layers 'layer.1;layer.2;layer.3' -python make_adapter.py --num_epochs 15 --data_path '../sharedtask2023/data' --freeze_layers 'layer.1' +python make_adapter.py --num_epochs 15 --data_path 'PATH_TO_REPO' --freeze_layers 'layer.1;layer.2;layer.3' +python make_adapter.py --num_epochs 15 --data_path 'PATH_TO_REPO' --freeze_layers 'layer.1' # Run classifiers with adapters -python adapter_classifier.py --num_epochs 3 --data_path '../sharedtask2023/data' --adapter_name 'adapter_15-epochs_frozen-1' -python adapter_classifier.py --num_epochs 4 --data_path '../sharedtask2023/data' --adapter_name 'adapter_15-epochs_frozen-1-2-3' \ No newline at end of file +python adapter_classifier.py --num_epochs 3 --data_path 'PATH_TO_REPO' --adapter_name 'adapter_15-epochs_frozen-1' +python adapter_classifier.py --num_epochs 4 --data_path 'PATH_TO_REPO' --adapter_name 'adapter_15-epochs_frozen-1-2-3' \ No newline at end of file