diff --git a/adapter_classifier.py b/adapter_classifier.py
index a260108fd33a183b38166e0b4852f7a24628c47e..28eb3b131afe8850da00edc504dfce4d22b00e5b 100644
--- a/adapter_classifier.py
+++ b/adapter_classifier.py
@@ -30,11 +30,11 @@ substitutions_file = 'mappings/substitutions.txt'
tokenizer = AutoTokenizer.from_pretrained(args.transformer_model)
# we are saving the test results of specific epochs
-specific_results = open_specific_results('mappings/specific_results.txt')
-if '1-2-3' in adapter_name or 'layer1;layer2;layer3' in adapter_name:
- specific_results = list(specific_results['A1_3'][args.num_epochs])
-else:
- specific_results = list(specific_results['A1'][args.num_epochs])
+# specific_results = open_specific_results('mappings/specific_results.txt')
+# if '1-2-3' in adapter_name or 'layer1;layer2;layer3' in adapter_name:
+# specific_results = list(specific_results['A1_3'][args.num_epochs])
+# else:
+# specific_results = list(specific_results['A1'][args.num_epochs])
set_seed(42)
@@ -65,8 +65,7 @@ dev_dict_df = {corpus : pd.DataFrame([[' '.join(x[-2]), x[-1]]
test_dict_df = {corpus : pd.DataFrame([[' '.join(x[-2]), x[-1]]
for x in sents],
columns = file_header)
- for corpus, sents in test_dict_sentences.items()
- if corpus in specific_results}
+ for corpus, sents in test_dict_sentences.items()}
#Make datasets from dataframes
train_dataset = datasets.Dataset.from_pandas(train_df)
@@ -96,7 +95,7 @@ for corpus in test_dict_dataset:
encoded_test_dataset[corpus] = temp
# ===============================
-# ## Training params
+# Training params
# ===============================
model = AutoAdapterModel.from_pretrained(args.transformer_model)
@@ -144,15 +143,72 @@ trainer.train()
print('\nDev results:')
for corpus in encoded_dev_dataset:
print()
- _ = get_predictions_huggingface(trainer, corpus,
+ dev_results = get_predictions_huggingface(trainer, corpus,
encoded_dev_dataset[corpus])
-# Save specific test results
-
-print('\nTest results:')
+
+
+ path_results = 'results/dev/' + adapter_name + '_' + str(args.num_epochs)
+ if not os.path.exists(path_results):
+ os.makedirs(path_results)
+
+ print_results_to_file(corpus,
+ dev_dict_sentences[corpus],
+ dev_results,
+ inv_mappings,
+ substitutions_file,
+ path_results)
+
+# Test results
+
+print('\ntest results:')
for corpus in encoded_test_dataset:
print()
- test_results = get_predictions_huggingface(trainer, corpus,
- encoded_test_dataset[corpus])
+ test_results = get_predictions_huggingface(trainer,
+ corpus,
+ encoded_test_dataset[corpus])
+
+
+ path_results = 'results/test/' + adapter_name + '_' + str(args.num_epochs)
+ if not os.path.exists(path_results):
+ os.makedirs(path_results)
+
+ print_results_to_file(corpus,
+ test_dict_sentences[corpus],
+ test_results,
+ inv_mappings,
+ substitutions_file,
+ path_results)
+
+
+
+# for corpus in test_dict_dataloader:
+# test_results = get_predictions(model,
+# corpus,
+# test_dict_dataloader[corpus])
+
+# path_results = 'results/test/pytorch' + str(epoch_num+1)
+# if not os.path.exists(path_results):
+# os.makedirs(path_results)
+
+# print_results_to_file(corpus,
+# test_dict_sentences[corpus],
+# test_results,
+# inv_mappings, substitutions_file,
+# path_results)
+
+
+
+
+
+
+
+# Save specific test results
- print_results_to_file(corpus, test_dict_sentences[corpus], test_results,
- inv_mappings, substitutions_file)
\ No newline at end of file
+# print('\nTest results:')
+# for corpus in encoded_test_dataset:
+# print()
+# test_results = get_predictions_huggingface(trainer, corpus,
+# encoded_test_dataset[corpus])
+#
+# print_results_to_file(corpus, test_dict_sentences[corpus], test_results,
+# inv_mappings, substitutions_file)
\ No newline at end of file
diff --git a/check_labels.ipynb b/check_labels.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..1305fdb3572d3dcd57d1af2cbf904babfd1338df
--- /dev/null
+++ b/check_labels.ipynb
@@ -0,0 +1,288 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "num_labels = 134 + 1\n",
+ "\n",
+ "temp = {}\n",
+ "mappings = {}\n",
+ "subs = {}\n",
+ "\n",
+ "\n",
+ "with open('mappings/mappings_substitutions.tsv', 'r') as f:\n",
+ " counter = -1\n",
+ " for line in f:\n",
+ " counter += 1\n",
+ " if counter < num_labels:\n",
+ " mappings[line.split(\"\\t\")[0]] = int(line.strip().split(\"\\t\")[1])\n",
+ " else:\n",
+ " temp[line.split(\"\\t\")[0]] = int(line.strip().split(\"\\t\")[1])\n",
+ " \n",
+ "inv_mappings = {v:k for k, v in mappings.items()}\n",
+ "subs = {k:inv_mappings[v] for k, v in temp.items()}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def read_corpus(file):\n",
+ " labels = []\n",
+ "\n",
+ " with open(file, 'r') as f:\n",
+ " next(f)\n",
+ " for line in f:\n",
+ " labels.append(line.strip().split('\\t')[-1])\n",
+ " return labels"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path = '/users/melodi/emetheni/clean_data'\n",
+ "list_corpora = [x for x in os.listdir(path)]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "\n",
+ "==spa.rst.sctb==\n",
+ "['antithesis', 'attribution', 'background', 'cause', 'circumstance', 'concession', 'condition', 'conjunction', 'contrast', 'disjunction', 'elaboration', 'evaluation', 'evidence', 'interpretation', 'justify', 'list', 'means', 'motivation', 'preparation', 'purpose', 'restatement', 'result', 'sequence', 'summary']\n",
+ "\n",
+ "\n",
+ "==tha.pdtb.tdtb==\n",
+ "['Comparison.Concession', 'Comparison.Contrast', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Cause+Belief', 'Contingency.Cause+SpeechAct', 'Contingency.Condition', 'Contingency.Negative-Condition', 'Contingency.Negative-Condition+SpeechAct', 'Contingency.Purpose', 'Expansion.Conjunction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Exception', 'Expansion.GenExpansion', 'Expansion.Instantiation', 'Expansion.Level-of-detail', 'Expansion.Substitution', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "\n",
+ "\n",
+ "==rus.rst.rrt==\n",
+ "['antithesis', 'attribution', 'background', 'cause', 'cause-effect', 'comparison', 'concession', 'conclusion', 'condition', 'contrast', 'effect', 'elaboration', 'evaluation', 'evidence', 'interpretation-evaluation', 'joint', 'motivation', 'preparation', 'purpose', 'restatement', 'sequence', 'solutionhood']\n",
+ "\n",
+ "\n",
+ "==zho.rst.gcdt==\n",
+ "['adversative-antithesis', 'adversative-concession', 'adversative-contrast', 'attribution-negative', 'attribution-positive', 'causal-cause', 'causal-result', 'context-background', 'context-circumstance', 'contingency-condition', 'elaboration-additional', 'elaboration-attribute', 'evaluation-comment', 'explanation-evidence', 'explanation-justify', 'explanation-motivation', 'joint-disjunction', 'joint-list', 'joint-other', 'joint-sequence', 'mode-manner', 'mode-means', 'organization-heading', 'organization-phatic', 'organization-preparation', 'purpose-attribute', 'purpose-goal', 'restatement-partial', 'restatement-repetition', 'topic-question', 'topic-solutionhood']\n",
+ "\n",
+ "\n",
+ "==fra.sdrt.annodis==\n",
+ "['alternation', 'attribution', 'background', 'comment', 'conditional', 'continuation', 'contrast', 'e-elaboration', 'elaboration', 'explanation', 'explanation*', 'flashback', 'frame', 'goal', 'narration', 'parallel', 'result', 'temploc']\n",
+ "\n",
+ "\n",
+ "==por.rst.cstn==\n",
+ "['antithesis', 'attribution', 'background', 'circumstance', 'comparison', 'concession', 'conclusion', 'condition', 'contrast', 'elaboration', 'enablement', 'evaluation', 'evidence', 'explanation', 'interpretation', 'joint', 'justify', 'list', 'means', 'motivation', 'nonvolitional-cause', 'nonvolitional-cause-e', 'nonvolitional-result', 'nonvolitional-result-e', 'otherwise', 'parenthetical', 'purpose', 'restatement', 'sequence', 'solutionhood', 'volitional-cause', 'volitional-result']\n",
+ "\n",
+ "\n",
+ "==eng.sdrt.stac==\n",
+ "['Acknowledgement', 'Alternation', 'Background', 'Clarification_question', 'Comment', 'Conditional', 'Continuation', 'Contrast', 'Correction', 'Elaboration', 'Explanation', 'Narration', 'Parallel', 'Q_Elab', 'Question_answer_pair', 'Result']\n",
+ "\n",
+ "\n",
+ "==eus.rst.ert==\n",
+ "['antithesis', 'background', 'cause', 'circumstance', 'concession', 'condition', 'conjunction', 'contrast', 'disjunction', 'elaboration', 'enablement', 'evaluation', 'evidence', 'interpretation', 'joint', 'justify', 'list', 'means', 'motivation', 'otherwise', 'preparation', 'purpose', 'restatement', 'result', 'sequence', 'solutionhood', 'summary', 'unconditional', 'unless']\n",
+ "\n",
+ "\n",
+ "==zho.dep.scidtb==\n",
+ "['ROOT', 'attribution', 'bg-compare', 'bg-general', 'bg-goal', 'cause', 'comparison', 'condition', 'contrast', 'elab-addition', 'elab-aspect', 'elab-enumember', 'elab-process_step', 'enablement', 'evaluation', 'exp-evidence', 'exp-reason', 'joint', 'manner-means', 'progression', 'result', 'summary', 'temporal']\n",
+ "\n",
+ "\n",
+ "==eng.pdtb.pdtb==\n",
+ "['Comparison.Concession', 'Comparison.Concession+SpeechAct', 'Comparison.Contrast', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Cause+Belief', 'Contingency.Cause+SpeechAct', 'Contingency.Condition', 'Contingency.Condition+SpeechAct', 'Contingency.Negative-cause', 'Contingency.Negative-condition', 'Contingency.Purpose', 'Expansion.Conjunction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Exception', 'Expansion.Instantiation', 'Expansion.Level-of-detail', 'Expansion.Manner', 'Expansion.Substitution', 'Hypophora', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "topic eng.pdtb.pdtb \n",
+ "\n",
+ "\n",
+ "==deu.rst.pcc==\n",
+ "['antithesis', 'background', 'cause', 'circumstance', 'concession', 'condition', 'conjunction', 'contrast', 'disjunction', 'e-elaboration', 'elaboration', 'evaluation-n', 'evaluation-s', 'evidence', 'interpretation', 'joint', 'list', 'means', 'preparation', 'purpose', 'reason', 'restatement', 'result', 'sequence', 'solutionhood', 'summary']\n",
+ "\n",
+ "\n",
+ "==eng.rst.rstdt==\n",
+ "['attribution', 'background', 'cause', 'comparison', 'condition', 'contrast', 'elaboration', 'enablement', 'evaluation', 'explanation', 'joint', 'manner-means', 'summary', 'temporal', 'textual-organization', 'topic-change', 'topic-comment']\n",
+ "acknowledgement eng.rst.rstdt \n",
+ "\n",
+ "\n",
+ "==zho.rst.sctb==\n",
+ "['antithesis', 'attribution', 'background', 'cause', 'circumstance', 'concession', 'condition', 'conjunction', 'contrast', 'disjunction', 'elaboration', 'enablement', 'evaluation', 'evidence', 'interpretation', 'justify', 'list', 'means', 'motivation', 'preparation', 'purpose', 'restatement', 'result', 'sequence', 'solutionhood', 'summary']\n",
+ "\n",
+ "\n",
+ "==nld.rst.nldt==\n",
+ "['antithesis', 'background', 'circumstance', 'concession', 'condition', 'conjunction', 'contrast', 'disjunction', 'elaboration', 'enablement', 'evaluation', 'evidence', 'interpretation', 'joint', 'justify', 'list', 'means', 'motivation', 'nonvolitional-cause', 'nonvolitional-result', 'otherwise', 'preparation', 'purpose', 'restatement', 'restatement-mn', 'sequence', 'solutionhood', 'summary', 'unconditional', 'unless', 'volitional-cause', 'volitional-result']\n",
+ "\n",
+ "\n",
+ "==tur.pdtb.tdb==\n",
+ "['Comparison.Concession', 'Comparison.Concession+SpeechAct', 'Comparison.Contrast', 'Comparison.Degree', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Cause+Belief', 'Contingency.Cause+SpeechAct', 'Contingency.Condition', 'Contingency.Negative-condition', 'Contingency.Purpose', 'Expansion.Conjunction', 'Expansion.Correction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Exception', 'Expansion.Instantiation', 'Expansion.Level-of-detail', 'Expansion.Manner', 'Expansion.Substitution', 'Hypophora', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "\n",
+ "\n",
+ "==spa.rst.rststb==\n",
+ "['alternative', 'antithesis', 'background', 'cause', 'circumstance', 'concession', 'condition', 'conjunction', 'contrast', 'disjunction', 'elaboration', 'enablement', 'evaluation', 'evidence', 'interpretation', 'joint', 'justify', 'list', 'means', 'motivation', 'preparation', 'purpose', 'restatement', 'result', 'sequence', 'solutionhood', 'summary', 'unless']\n",
+ "\n",
+ "\n",
+ "==por.pdtb.tedm==\n",
+ "['Comparison.Concession', 'Comparison.Contrast', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Cause+Belief', 'Contingency.Condition', 'Contingency.Condition+SpeechAct', 'Contingency.Purpose', 'Expansion.Conjunction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Instantiation', 'Expansion.Level-of-detail', 'Expansion.Manner', 'Expansion.Substitution', 'Hypophora', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "\n",
+ "\n",
+ "==ita.pdtb.luna==\n",
+ "['', 'Comparison', 'Comparison.Concession', 'Comparison.Contrast', 'Contingency.Cause', 'Contingency.Condition', 'Contingency.Goal', 'Expansion.Alternative', 'Expansion.Conjunction', 'Expansion.Instantiation', 'Expansion.Restatement', 'Interrupted', 'Repetition', 'Temporal.Asynchronous', 'Temporal.Synchrony']\n",
+ "parallel ita.pdtb.luna \n",
+ "\n",
+ "\n",
+ "==fas.rst.prstc==\n",
+ "['attribution', 'background', 'cause', 'comparison', 'condition', 'contrast', 'elaboration', 'enablement', 'evaluation', 'explanation', 'joint', 'manner-means', 'summary', 'temporal', 'topic-change', 'topic-comment', 'topic-drift']\n",
+ "\n",
+ "\n",
+ "==por.pdtb.crpc==\n",
+ "['Comparison', 'Comparison.Concession', 'Comparison.Contrast', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Condition', 'Contingency.Negative', 'Contingency.Purpose', 'Expansion.Conjunction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Exception', 'Expansion.Instantiation', 'Expansion.Level', 'Expansion.Manner', 'Expansion.Substitution', 'Hypophora', 'QAP', 'QAP.Hypophora', 'Temporal', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "\n",
+ "\n",
+ "==zho.pdtb.cdtb==\n",
+ "['Alternative', 'Causation', 'Conditional', 'Conjunction', 'Contrast', 'Expansion', 'Progression', 'Purpose', 'Temporal']\n",
+ "preparation zho.pdtb.cdtb \n",
+ "\n",
+ "\n",
+ "==eng.pdtb.tedm==\n",
+ "['Comparison.Concession', 'Comparison.Contrast', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Cause+Belief', 'Contingency.Cause+SpeechAct', 'Contingency.Condition', 'Contingency.Purpose', 'Expansion.Conjunction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Instantiation', 'Expansion.Level-of-detail', 'Expansion.Manner', 'Expansion.Substitution', 'Hypophora', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "parallel eng.pdtb.tedm \n",
+ "\n",
+ "\n",
+ "==eng.rst.gum==\n",
+ "['adversative', 'attribution', 'causal', 'context', 'contingency', 'elaboration', 'evaluation', 'explanation', 'joint', 'mode', 'organization', 'purpose', 'restatement', 'topic']\n",
+ "attribution-positive eng.rst.gum \n",
+ "\n",
+ "\n",
+ "==eng.dep.covdtb==\n",
+ "['ATTRIBUTION', 'BACKGROUND', 'CAUSE-RESULT', 'COMPARISON', 'CONDITION', 'ELABORATION', 'ENABLEMENT', 'FINDINGS', 'JOINT', 'MANNER-MEANS', 'TEMPORAL', 'TEXTUAL-ORGANIZATION']\n",
+ "interpretation eng.dep.covdtb \n",
+ "\n",
+ "\n",
+ "==tur.pdtb.tedm==\n",
+ "['Comparison.Concession', 'Comparison.Concession+SpeechAct', 'Comparison.Contrast', 'Comparison.Similarity', 'Contingency.Cause', 'Contingency.Cause+Belief', 'Contingency.Cause+SpeechAct', 'Contingency.Condition', 'Contingency.Negative-condition', 'Contingency.Purpose', 'Expansion', 'Expansion.Conjunction', 'Expansion.Disjunction', 'Expansion.Equivalence', 'Expansion.Exception', 'Expansion.Instantiation', 'Expansion.Level-of-detail', 'Expansion.Manner', 'Expansion.Substitution', 'Hypophora', 'Temporal.Asynchronous', 'Temporal.Synchronous']\n",
+ "\n",
+ "\n",
+ "==eng.dep.scidtb==\n",
+ "['attribution', 'bg-compare', 'bg-general', 'bg-goal', 'cause', 'comparison', 'condition', 'contrast', 'elab-addition', 'elab-aspect', 'elab-definition', 'elab-enumember', 'elab-example', 'elab-process_step', 'enablement', 'evaluation', 'exp-evidence', 'exp-reason', 'joint', 'manner-means', 'progression', 'result', 'summary', 'temporal']\n"
+ ]
+ }
+ ],
+ "source": [
+ "for corpus in list_corpora:\n",
+ " \n",
+ " labels = read_corpus(path + '/' + corpus + '/' + corpus + '_dev.rels')\n",
+ " labels = read_corpus(path + '/' + corpus + '/' + corpus + '_test.rels')\n",
+ " try:\n",
+ " labels = read_corpus(path + '/' + corpus + '/' + corpus + '_train.rels')\n",
+ " except:\n",
+ " pass\n",
+ "\n",
+ " labels = set(labels)\n",
+ " print('\\n')\n",
+ " print(\"==\" + corpus + \"==\")\n",
+ " print(sorted(labels))\n",
+ "\n",
+ "# for label in labels:\n",
+ "# if label not in mappings:\n",
+ "# if label.lower() in mappings:\n",
+ "# print(label + '\\t' + corpus + '\\t' + label.lower())\n",
+ "# elif label in subs:\n",
+ "# print(label + '\\t' + corpus + '\\t' + subs[label])\n",
+ "# elif label.lower() in subs:\n",
+ "# print(label + '\\t' + corpus + '\\t' + subs[label.lower()])\n",
+ "# else:\n",
+ "# print('AAAAAAAAAAAAAAAAAAAA', label, corpus)\n",
+ "\n",
+ "\n",
+ " test_labels = read_corpus('results/test/A_15-epochs_frozen-1_3/' + corpus +'.tsv')\n",
+ " test_labels = set(test_labels)\n",
+ " \n",
+ " for l in test_labels:\n",
+ " if l not in labels:\n",
+ " temp = ''\n",
+ " \n",
+ " if l.lower() in labels:\n",
+ " temp = l.lower()\n",
+ " elif l.lower() in inv_mappings:\n",
+ " temp = inv_mappings[l.lower()]\n",
+ " elif l in subs:\n",
+ " temp = subs[l]\n",
+ " elif l.lower() in subs:\n",
+ " temp = subs[l.lower()]\n",
+ " \n",
+ " try:\n",
+ " assert temp in test_labels\n",
+ " print(l + ' ' + corpus + ' ' +temp)\n",
+ " except: \n",
+ " print(l + ' ' + corpus + ' ' +temp)\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/huggingface-classifier.py b/huggingface-classifier.py
new file mode 100644
index 0000000000000000000000000000000000000000..28eb3b131afe8850da00edc504dfce4d22b00e5b
--- /dev/null
+++ b/huggingface-classifier.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import torch
+import numpy as np
+from transformers import AutoModel, AutoTokenizer, get_linear_schedule_with_warmup, AutoAdapterModel, AutoModelWithHeads, AutoConfig, TrainingArguments, Trainer, EvalPrediction, set_seed
+from torch import nn
+from torch.optim import AdamW
+from torch.utils.data import DataLoader
+import torch.nn.functional as F
+from torch.autograd import Variable
+from tqdm import tqdm
+import os
+from time import sleep
+from datetime import datetime
+import sys
+from sklearn.metrics import classification_report, accuracy_score
+from utils import open_file
+import pandas as pd
+import datasets
+from configure import parse_args
+from utils import *
+
+args = parse_args()
+now = datetime.now()
+dt_string = now.strftime("%d.%m.%y-%H:%M:%S")
+adapter_name = args.adapter_name
+mappings, inv_mappings = open_mappings(args.mappings_file)
+substitutions_file = 'mappings/substitutions.txt'
+tokenizer = AutoTokenizer.from_pretrained(args.transformer_model)
+
+# we are saving the test results of specific epochs
+# specific_results = open_specific_results('mappings/specific_results.txt')
+# if '1-2-3' in adapter_name or 'layer1;layer2;layer3' in adapter_name:
+# specific_results = list(specific_results['A1_3'][args.num_epochs])
+# else:
+# specific_results = list(specific_results['A1'][args.num_epochs])
+
+set_seed(42)
+
+print('Train classifier with adapter\n')
+print('Adapter name:', adapter_name)
+print('Model:', args.transformer_model)
+print('Batch size:', args.batch_size * args.gradient_accumulation_steps)
+print('Num epochs:', args.num_epochs)
+
+# Open mappings
+mappings, inv_mappings = open_mappings(args.mappings_file)
+
+# Open sentences
+train_sentences, dev_dict_sentences, test_dict_sentences = open_sentences(args.data_path, mappings)
+
+# make pandas dataframes
+file_header = ['text', 'labels']
+
+train_df = pd.DataFrame([[' '.join(x[-2]), x[-1]] for x in train_sentences],
+ columns =file_header)
+train_df = train_df.sample(frac = 1) # shuffle the train
+
+dev_dict_df = {corpus : pd.DataFrame([[' '.join(x[-2]), x[-1]]
+ for x in sents],
+ columns = file_header)
+ for corpus, sents in dev_dict_sentences.items()}
+
+test_dict_df = {corpus : pd.DataFrame([[' '.join(x[-2]), x[-1]]
+ for x in sents],
+ columns = file_header)
+ for corpus, sents in test_dict_sentences.items()}
+
+#Make datasets from dataframes
+train_dataset = datasets.Dataset.from_pandas(train_df)
+dev_dict_dataset = {corpus:datasets.Dataset.from_pandas(dev_df)
+ for corpus, dev_df in dev_dict_df.items()}
+test_dict_dataset = {corpus:datasets.Dataset.from_pandas(dev_df)
+ for corpus, dev_df in test_dict_df.items()}
+
+# get number of labels
+num_labels = len(set([int(x.strip())
+ for x in train_df['labels'].to_string(index=False).split('\n')])) +1
+
+# Encode the data
+train_dataset = train_dataset.map(encode_batch, batched=True)
+train_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+encoded_dev_dataset = {}
+for corpus in dev_dict_dataset:
+ temp = dev_dict_dataset[corpus].map(encode_batch, batched=True)
+ temp.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+ encoded_dev_dataset[corpus] = temp
+
+encoded_test_dataset = {}
+for corpus in test_dict_dataset:
+ temp = test_dict_dataset[corpus].map(encode_batch, batched=True)
+ temp.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+ encoded_test_dataset[corpus] = temp
+
+# ===============================
+# Training params
+# ===============================
+
+model = AutoAdapterModel.from_pretrained(args.transformer_model)
+active_adapter = model.load_adapter(adapter_name,
+ config = adapter_name + "/adapter_config.json")
+model.set_active_adapters(active_adapter)
+
+
+training_args = TrainingArguments(
+ learning_rate = 2e-5, #1e-4,
+ num_train_epochs = args.num_epochs,
+ per_device_train_batch_size = args.batch_size,
+ per_device_eval_batch_size = args.batch_size,
+ gradient_accumulation_steps = args.gradient_accumulation_steps,
+ logging_steps = (len(train_sentences)/(args.batch_size * args.gradient_accumulation_steps)),
+ output_dir = "./training_output",
+ overwrite_output_dir =True,
+ remove_unused_columns=False,
+)
+
+
+trainer = Trainer(
+ model = model,
+ args = training_args,
+ train_dataset = train_dataset
+)
+
+# Freeze layers in the classifier if desired
+if args.freeze_layers != '':
+ layers_to_freeze = args.freeze_layers.split(';')
+ for name, param in model.named_parameters():
+ if any(x in name for x in layers_to_freeze):
+ param.requires_grad = False
+
+
+# ===============================
+# Start the training 🚀
+# ===============================
+
+print('Start training...')
+trainer.train()
+
+# Dev results
+
+print('\nDev results:')
+for corpus in encoded_dev_dataset:
+ print()
+ dev_results = get_predictions_huggingface(trainer, corpus,
+ encoded_dev_dataset[corpus])
+
+
+ path_results = 'results/dev/' + adapter_name + '_' + str(args.num_epochs)
+ if not os.path.exists(path_results):
+ os.makedirs(path_results)
+
+ print_results_to_file(corpus,
+ dev_dict_sentences[corpus],
+ dev_results,
+ inv_mappings,
+ substitutions_file,
+ path_results)
+
+# Test results
+
+print('\ntest results:')
+for corpus in encoded_test_dataset:
+ print()
+ test_results = get_predictions_huggingface(trainer,
+ corpus,
+ encoded_test_dataset[corpus])
+
+
+ path_results = 'results/test/' + adapter_name + '_' + str(args.num_epochs)
+ if not os.path.exists(path_results):
+ os.makedirs(path_results)
+
+ print_results_to_file(corpus,
+ test_dict_sentences[corpus],
+ test_results,
+ inv_mappings,
+ substitutions_file,
+ path_results)
+
+
+
+# for corpus in test_dict_dataloader:
+# test_results = get_predictions(model,
+# corpus,
+# test_dict_dataloader[corpus])
+
+# path_results = 'results/test/pytorch' + str(epoch_num+1)
+# if not os.path.exists(path_results):
+# os.makedirs(path_results)
+
+# print_results_to_file(corpus,
+# test_dict_sentences[corpus],
+# test_results,
+# inv_mappings, substitutions_file,
+# path_results)
+
+
+
+
+
+
+
+# Save specific test results
+
+# print('\nTest results:')
+# for corpus in encoded_test_dataset:
+# print()
+# test_results = get_predictions_huggingface(trainer, corpus,
+# encoded_test_dataset[corpus])
+#
+# print_results_to_file(corpus, test_dict_sentences[corpus], test_results,
+# inv_mappings, substitutions_file)
\ No newline at end of file
diff --git a/make_adapter.py b/make_adapter.py
index 71bfca21e0ef5ce0a5913965d9dd02d734f16e15..d51875ca89cfdbd52730e8248f4a7fe5add3d24d 100644
--- a/make_adapter.py
+++ b/make_adapter.py
@@ -20,7 +20,7 @@ set_seed(42)
batch_size = args.batch_size
# Set name for adapter
-adapter_name = 'adapter_' + str(args.num_epochs) + '-epochs_frozen' + args.freeze_layers.replace('layer.', '-').replace(';', '')
+adapter_name = 'A_' + str(args.num_epochs) + '-epochs_frozen' + args.freeze_layers.replace('layer.', '-').replace(';', '')
print('Create classifier adapter\n')
print('Name:', adapter_name)
diff --git a/pytorch_classifier.py b/pytorch_classifier.py
index ad32fc783ed88c7133f7ff3a7cda56a6a02e0e3c..a86e6a9bb450e819e0aec4d7c53ead8b9a62662d 100644
--- a/pytorch_classifier.py
+++ b/pytorch_classifier.py
@@ -166,7 +166,8 @@ def train(model,
total_loss_train = 0
batch_counter = 0
- for train_input, train_label in tqdm(train_dataloader):
+# for train_input, train_label in tqdm(train_dataloader):
+ for train_input, train_label in train_dataloader:
batch_counter += 1
train_label = train_label.to(device)
mask = train_input['attention_mask'].to(device)
@@ -201,27 +202,60 @@ def train(model,
scheduler.step()
# ------ Validation --------
+
print('\nValidation for epoch:', epoch_num + 1)
- # Dev results for each corpus. We don't need to save the results.
+ # Dev and test results for each corpus. We don't need to save the results.
for corpus in dev_dict_dataloader:
- _ = get_predictions(model,
+ dev_results = get_predictions(model,
corpus,
dev_dict_dataloader[corpus])
- # we want the results of specific epochs for specific corpora.
- # we define the epochs and the corpora and we save only these results.
+ path_results = 'results/dev/pytorch-' + str(epoch_num+1)
+ if not os.path.exists(path_results):
+ os.makedirs(path_results)
+
+ print_results_to_file(corpus,
+ dev_dict_sentences[corpus],
+ dev_results,
+ inv_mappings, substitutions_file,
+ path_results)
+
+ # ------ Test --------
+
+ print('\nTest results for epoch:', epoch_num + 1)
+
+ for corpus in test_dict_dataloader:
+ test_results = get_predictions(model,
+ corpus,
+ test_dict_dataloader[corpus])
+
+ path_results = 'results/test/pytorch' + str(epoch_num+1)
+ if not os.path.exists(path_results):
+ os.makedirs(path_results)
+
+ print_results_to_file(corpus,
+ test_dict_sentences[corpus],
+ test_results,
+ inv_mappings, substitutions_file,
+ path_results)
+
+
+# # we want the results of specific epochs for specific corpora.
+# # we define the epochs and the corpora and we save only these results.
+
+# if epoch_num+1 in specific_results:
+# for corpus in specific_results[epoch_num+1]:
+# test_results = get_predictions(model,
+# corpus,
+# test_dict_dataloader[corpus],
+# print_results=False)
+
+
+ # ========= New Code! =============
+ # Save for each epoch the dev and test results
+
- if epoch_num+1 in specific_results:
- for corpus in specific_results[epoch_num+1]:
- test_results = get_predictions(model,
- corpus,
- test_dict_dataloader[corpus],
- print_results=False)
- print_results_to_file(corpus,
- test_dict_sentences[corpus],
- test_results,
- inv_mappings, substitutions_file)
# ------- Start the training -------
@@ -241,12 +275,12 @@ print('\nTraining Done!')
# ------- Testing ---------
-print('Testing...')
-for corpus in test_dict_dataloader:
- test_results = get_predictions(model,
- corpus,
- test_dict_dataloader[corpus]
- )
+# print('Testing...')
+# for corpus in test_dict_dataloader:
+# test_results = get_predictions(model,
+# corpus,
+# test_dict_dataloader[corpus]
+# )
# print_results_to_file(corpus,
# test_dict_sentences[corpus],
# test_results,
diff --git a/requirements.txt b/requirements.txt
index e03aac89c19c88e5496beee1eb6ce1f781c316b4..396a0f04275209d4371e47b3b7fc448f1de217b3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,17 +1,9 @@
adapter-transformers==3.0.1
-aiohttp==3.8.4
-aiosignal==1.3.1
-async-timeout==4.0.2
-attrs==23.1.0
certifi==2023.5.7
-charset-normalizer==3.1.0
-click==8.1.3
+charset-normalizer
cmake==3.26.3
datasets==2.4.0
-dill==0.3.5.1
-filelock==3.12.0
-frozenlist==1.3.3
-fsspec==2023.5.0
+fsspec
huggingface-hub==0.14.1
idna==3.4
Jinja2==3.1.2
@@ -22,18 +14,6 @@ mpmath==1.3.0
multidict==6.0.4
multiprocess==0.70.13
networkx==3.1
-numpy==1.24.3
-nvidia-cublas-cu11==11.10.3.66
-nvidia-cuda-cupti-cu11==11.7.101
-nvidia-cuda-nvrtc-cu11==11.7.99
-nvidia-cuda-runtime-cu11==11.7.99
-nvidia-cudnn-cu11==8.5.0.96
-nvidia-cufft-cu11==10.9.0.58
-nvidia-curand-cu11==10.2.10.91
-nvidia-cusolver-cu11==11.4.0.1
-nvidia-cusparse-cu11==11.7.4.91
-nvidia-nccl-cu11==2.14.3
-nvidia-nvtx-cu11==11.7.91
packaging==23.1
pandas==2.0.1
Pillow==9.5.0
@@ -53,7 +33,7 @@ threadpoolctl==3.1.0
tokenizers==0.12.1
torch==2.0.1
torchaudio==2.0.2
-torchvision==0.15.2
+torchvision
tqdm==4.65.0
transformers==4.18.0
triton==2.0.0
diff --git a/run_stuff.sh b/run_stuff.sh
new file mode 100644
index 0000000000000000000000000000000000000000..83d9213770e9279e0a1b9034e8e6430e95145ef9
--- /dev/null
+++ b/run_stuff.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+#SBATCH --job-name=finals
+
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --partition=GPUNodes
+#SBATCH --gres=gpu:1
+
+
+# tests tests
+
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 pytorch_classifier.py --batch_size 8 --num_epochs 6 --data_path '/users/melodi/emetheni/clean_data'
+
+# Train the adapter:
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 make_adapter.py --batch_size 8 --num_epochs 15 --data_path '/users/melodi/emetheni/sharedtask2023/data' --freeze_layers 'layer.1;layer.2;layer.3'
+
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 make_adapter.py --batch_size 8 --num_epochs 15 --data_path '/users/melodi/emetheni/sharedtask2023/data' --freeze_layers 'layer.1'
+
+# Run classifier with adapter for corpora:
+
+
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 1 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1-2-3'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 2 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1-2-3'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 3 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1-2-3'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 4 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1-2-3'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 5 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1-2-3'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 6 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1-2-3'
+
+
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 1 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 2 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1'
+srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 3 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 4 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 5 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1'
+# srun singularity exec /logiciels/containerCollections/CUDA10/pytorch.sif python3 adapter_classifier.py --batch_size 8 --num_epochs 6 --data_path '/users/melodi/emetheni/clean_data' --adapter_name 'A_15-epochs_frozen-1'
diff --git a/see_results.ipynb b/see_results.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..8d1d5dccb48ce9b9dd7a757b64f730c2bcd295d0
--- /dev/null
+++ b/see_results.ipynb
@@ -0,0 +1,411 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.metrics import accuracy_score\n",
+ "import os, io\n",
+ "from collections import OrderedDict, Counter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "connectives = {\"elaboration\": [\"and\", \"also\", \"besides\", \"further\", \"furthermore\", \"too\", \"moreover\", \"in addition\", \"then\", \"of equal importance\", \"equally important\", \"another\", \"additionally\", \"also\", \"moreover\", \"furthermore\", \"again\", \"further\", \"then\", \"besides\", \"too\", \"similarly\", \"correspondingly\", \"indeed\", \"regarding\"], \n",
+ "\"time\": [\"next\", \"afterward\", \"finally\", \"later\", \"last\", \"lastly\", \"at last\", \"now\", \"subsequently\", \"then\", \"when\", \"soon\", \"thereafter\", \"after a short time\", \"the next week\", \"a minute later\", \"in the meantime\", \"meanwhile\", \"on the following day\", \"at length\", \"ultimately\", \"presently\"], \n",
+ "\"sequence\": [\"first\", \"second\", \"third\", \"finally\", \"hence\", \"next\", \"then\", \"from here on\", \"to begin with\", \"last of all\", \"after\", \"before\", \"as soon as\", \"in the end\", \"gradually\", \"when\", \"after\", \"after that\", \"afterwards\", \"next\", \"subsequently\", \"later (on)\", \"followed by\", \"to go on to\", \"finally\", \"another\", \"additionally\", \"finally moreover\", \"also\", \"subsequently\", \"eventually\", \"next\", \"then\"], \n",
+ "\"example\": [\"for example\", \"to illustrate\", \"for instance\", \"to be specific\", \"such as\", \"moreover\", \"furthermore\", \"just as important\", \"similarly\", \"in the same way\", \"for example\", \"for instance\", \"namely\", \"such as\", \"as follows\", \"as exemplified by\", \"such as\", \"including\", \"especially\", \"particularly\", \"in particular\", \"notably\", \"mainly\"], \n",
+ "\"result\": [\"as a result\", \"hence\", \"so\", \"accordingly\", \"as a consequence\", \"consequently\", \"thus\", \"since\", \"therefore\", \"for this reason\", \"because of this\", \"therefore\", \"accordingly\", \"as a result of\", \"the result is/results are\", \"the consequence is\", \"resulting from\", \"consequently\", \"it can be seen\", \"evidence illustrates that\", \"because of this\", \"thus\", \"hence\", \"for this reason\", \"owing to x\", \"this suggests that\", \"it follows that\", \"otherwise\", \"in that case\", \"that implies\", \"As a result\", \"therefore\", \"thus\"], \n",
+ "\"purpose\": [\"for this purpose\", \"with this in mind\", \"for this reason\"], \n",
+ "\"comparison\": [\"like\", \"in the same manner\", \"as so\", \"similarly\"], \n",
+ "\"contrast\": [\"but\", \"in contrast\", \"conversely\", \"however\", \"still\", \"nevertheless\", \"nonetheless\", \"yet\", \"and yet\", \"on the other hand\", \"on the contrary\", \"or\", \"in spite of this\", \"actually\", \"in fact\", \"whereas\", \"conversely\", \"in comparison\", \"by contrast\", \"in contrast\", \"contrasting\", \"alternatively\", \"although\", \"otherwise\", \"instead\"], \n",
+ "\"summary\": [\"in summary\", \"to sum up\", \"to repeat\", \"briefly\", \"in short\", \"finally\", \"on the whole\", \"therefore\", \"as I have said\", \"in conclusion\", \"as seen\", \"in conclusion\", \"therefore\", \"to conclude\", \"on the whole\", \"hence\", \"thus to summarise\", \"altogether\", \"overall\"], \n",
+ "\"rephrasing\": [\"in other terms\", \"rather\", \"or\", \"better\", \"in view of this\", \"in contrast\"]}\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def parse_data(infile, string_input=False) -> list:\n",
+ " \"\"\"\n",
+ " This function is to read a gold or a pred file to obtain the label column for accuracy calculation.\n",
+ "\n",
+ " :param infile: shared task .rels file\n",
+ " :param string_input: If True, files are replaced by strings with file contents (for import inside other scripts)\n",
+ " :return: a list of labels\n",
+ " \"\"\"\n",
+ "\n",
+ " if not string_input:\n",
+ " data = io.open(infile, encoding=\"utf-8\").read().strip().replace(\"\\r\", \"\")\n",
+ " else:\n",
+ " data = infile.strip()\n",
+ "\n",
+ " labels = [line.split(\"\\t\")[-1].lower() \n",
+ " for i, line in enumerate(data.split(\"\\n\")) if \"\\t\" in line and i>0]\n",
+ " \n",
+ " sentences = [(line.split(\"\\t\")[3], line.split(\"\\t\")[4], line.split(\"\\t\")[-3])\n",
+ " for i, line in enumerate(data.split(\"\\n\")) if \"\\t\" in line and i>0]\n",
+ " \n",
+ " return sentences, labels\n",
+ "\n",
+ "\n",
+ "def get_accuracy_score(gold_file, pred_file, string_input=False) -> dict:\n",
+ "\n",
+ " _, gold_labels = parse_data(gold_file, string_input)\n",
+ " _, pred_labels = parse_data(pred_file, string_input)\n",
+ "\n",
+ " filename = gold_file.split(os.sep)[-1]\n",
+ "\n",
+ " assert len(gold_labels) == len(pred_labels), \"FATAL: different number of labels detected in gold and pred\"\n",
+ "\n",
+ " acc = accuracy_score(gold_labels, pred_labels)\n",
+ "\n",
+ " score_dict = {\"filename\": filename,\n",
+ " \"acc_score\": round(acc, 4),\n",
+ " \"gold_rel_count\": len(gold_labels),\n",
+ " \"pred_rel_count\": len(pred_labels)}\n",
+ "\n",
+ " return score_dict\n",
+ "\n",
+ "def separate_right_wrong(gold_file, pred_file, string_input=False):\n",
+ " \n",
+ " rights = []\n",
+ " wrongs = []\n",
+ " \n",
+ " gold_sents, gold_labels = parse_data(gold_file, string_input)\n",
+ " pred_sents, pred_labels = parse_data(pred_file, string_input)\n",
+ " \n",
+ " for n in range(len(gold_sents)):\n",
+ " if gold_labels[n] == pred_labels[n]:\n",
+ " rights.append([gold_sents[n], gold_labels[n], pred_labels[n]])\n",
+ " else:\n",
+ " wrongs.append([gold_sents[n], gold_labels[n], pred_labels[n]])\n",
+ " \n",
+ " return rights, wrongs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Print accuracies \n",
+ "\n",
+ "model = 'A_15-epochs_frozen-1_2'\n",
+ "corpus = 'eng.dep.covdtb'\n",
+ "\n",
+ "gold_path = '/users/melodi/emetheni/clean_data/'\n",
+ "results_path = 'results/test/' + model + '/'\n",
+ "\n",
+ "corpora = sorted([x[:-4] for x in os.listdir('results/test/' + model) \n",
+ " if not \"DS\" in x if not 'ipy' in x])\n",
+ "\n",
+ "# for corpus in corpora:\n",
+ "# score = get_accuracy_score(gold_path + corpus + '/' + corpus + '_test.rels', \n",
+ "# results_path + corpus + '.tsv')\n",
+ "\n",
+ "# print(corpus, '\\t', score['acc_score'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Separate\n",
+ "\n",
+ "# model = 'A_15-epochs_frozen-1_2'\n",
+ "# corpus = 'eng.dep.covdtb'\n",
+ "\n",
+ "model = 'A_15-epochs_frozen-1-2-3_3'\n",
+ "corpus = 'eng.rst.gum'\n",
+ "\n",
+ "gold_path = '/users/melodi/emetheni/clean_data/'\n",
+ "results_path = 'results/test/' + model + '/'\n",
+ "\n",
+ "corpora = sorted([x[:-4] for x in os.listdir('results/test/' + model) \n",
+ " if not \"DS\" in x if not 'ipy' in x])\n",
+ "\n",
+ "rights, wrongs = separate_right_wrong(gold_path + corpus + '/' + corpus + '_test.rels', \n",
+ " results_path + corpus + '.tsv')\n",
+ "\n",
+ "rights_count = dict(OrderedDict(Counter([x[-1] for x in rights])))\n",
+ "wrongs_count = dict(OrderedDict(Counter([x[-1] for x in wrongs])))\n",
+ "\n",
+ "# for label in sorted(set(list(rights_count.keys()) + list(wrongs_count.keys())), reverse=False):\n",
+ "# if label in rights_count:\n",
+ "# r = rights_count[label]\n",
+ "# else:\n",
+ "# r = 0\n",
+ "# if label in wrongs_count:\n",
+ "# w = wrongs_count[label]\n",
+ "# else:\n",
+ "# w = 0\n",
+ "# print(label, '\\t', r, '\\t', w)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "rights 203 / 1657 0.12251056125528063\n",
+ "wrongs 71 / 918 0.07734204793028322\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Presence of connectives in right/wrong sents\n",
+ "\n",
+ "counter = 0\n",
+ "for sent in rights:\n",
+ " sentence = (sent[0][0] + ' ' + sent[0][1]).lower()\n",
+ " if sent[1] in connectives:\n",
+ " if any(x in sentence for x in connectives[sent[1]]):\n",
+ "# print(sent)\n",
+ " counter += 1\n",
+ "print('rights', counter, '/', len(rights), counter/len(rights))\n",
+ "\n",
+ "counter = 0\n",
+ "for sent in wrongs:\n",
+ " \n",
+ " sentence = (sent[0][0] + ' ' + sent[0][1]).lower()\n",
+ " if sent[1] in connectives:\n",
+ " if any(x in sentence for x in connectives[sent[1]]):\n",
+ "# print(sent)\n",
+ " counter += 1\n",
+ "print('wrongs', counter, '/', len(wrongs), counter/len(wrongs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "rights 1253 / 1657 0.756185878092939\n",
+ "wrongs 735 / 918 0.8006535947712419\n"
+ ]
+ }
+ ],
+ "source": [
+ "# See direction\n",
+ "\n",
+ "counter = 0\n",
+ "for sent in rights:\n",
+ " if sent[0][2] == '1<2':\n",
+ " counter += 1\n",
+ "print('rights', counter, '/', len(rights), counter/len(rights))\n",
+ "\n",
+ "counter = 0\n",
+ "for sent in wrongs:\n",
+ " if sent[0][2] == '1<2':\n",
+ " counter += 1\n",
+ "print('wrongs', counter, '/', len(wrongs), counter/len(wrongs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[[('The prevalence of discrimination across racial groups in contemporary America :',\n",
+ " 'The current study seeks to build on this research',\n",
+ " '1>2'),\n",
+ " 'organization',\n",
+ " 'organization'],\n",
+ " [('The prevalence of discrimination across racial groups in contemporary America :',\n",
+ " 'Results from a nationally representative sample of adults',\n",
+ " '1<2'),\n",
+ " 'elaboration',\n",
+ " 'elaboration'],\n",
+ " [('Introduction .',\n",
+ " 'The current study seeks to build on this research',\n",
+ " '1>2'),\n",
+ " 'organization',\n",
+ " 'organization'],\n",
+ " [('Personal experiences of discrimination and bias have been the focus of much social science research .',\n",
+ " 'In many respects , researchers already possess a wealth of knowledge',\n",
+ " '1>2'),\n",
+ " 'context',\n",
+ " 'context'],\n",
+ " [('Personal experiences of discrimination and bias have been the focus of much social science research .',\n",
+ " '[ 1 - 3 ]',\n",
+ " '1<2'),\n",
+ " 'explanation',\n",
+ " 'explanation'],\n",
+ " [('Sociologists have explored the adverse consequences of discrimination',\n",
+ " '[ 3 – 5 ] ;',\n",
+ " '1<2'),\n",
+ " 'explanation',\n",
+ " 'explanation'],\n",
+ " [('Sociologists have explored the adverse consequences of discrimination',\n",
+ " 'psychologists have examined the mental processes',\n",
+ " '1<2'),\n",
+ " 'joint',\n",
+ " 'joint'],\n",
+ " [('psychologists have examined the mental processes',\n",
+ " 'that underpin conscious and unconscious biases',\n",
+ " '1<2'),\n",
+ " 'elaboration',\n",
+ " 'elaboration'],\n",
+ " [('psychologists have examined the mental processes', '[ 6 ] ;', '1<2'),\n",
+ " 'explanation',\n",
+ " 'explanation'],\n",
+ " [('Sociologists have explored the adverse consequences of discrimination',\n",
+ " 'neuroscientists have examined the neurobiological underpinnings of discrimination',\n",
+ " '1<2'),\n",
+ " 'joint',\n",
+ " 'joint']]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "rights[:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "subs = {\"Attribution\": [\"attribution\", \"attribution-negative\"],\n",
+ " \"Background\": [\"background\", \"circumstance\"],\n",
+ " \"Cause\": [\"cause\", \"result\", \"consequence\"],\n",
+ " \"Comparison\": [\"comparison\", \"preference\", \"analogy\", \"proportion\"],\n",
+ " \"Condition\": [\"condition\", \"hypothetical\", \"contingency\", \"otherwise\"],\n",
+ " \"Contrast\": [\"contrast\", \"concession\", \"antithesis\"],\n",
+ " \"Elaboration\": [\"elaboration-additional\", \"elaboration-general-specific\", \"elaboration-part-whole\", \"elaboration-process-step\", \"elaboration-object-attribute\", \"elaboration-set-member\", \"example\", \"definition\"],\n",
+ " \"Enablement\": [\"purpose\", \"enablement\"],\n",
+ " \"Evaluation\": [\"evaluation\", \"interpretation\", \"conclusion\", \"comment\"],\n",
+ " \"Explanation\": [\"evidence\", \"explanation-argumentative\", \"reason\"],\n",
+ " \"Joint\": [\"list\", \"disjunction\"],\n",
+ " \"Manner-Means\": [\"manner\", \"means\"],\n",
+ " \"Topic-Comment\": [\"problem-solution\", \"question-answer\", \"statement-response\", \"topic-comment\", \"comment-topic\", \"rhetorical-question\"],\n",
+ " \"Summary\": [\"summary\", \"restatement\"],\n",
+ " \"Temporal\": [\"temporal-before\", \"temporal-after\", \"temporal-same-time\", \"sequence\", \"inverted-sequence\"],\n",
+ " \"Topic Change\": [\"topic-shift\", \"topic-drift\"]}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rst = [\"bg-general\", \"elab-addition\", \"manner-means\", \"attribution\", \"evaluation\", \"enablement\", \"elab-aspect\", \"joint\", \"temporal\", \"result\", \"bg-goal\", \"progression\", \"contrast\", \"elab-process_step\", \"elab-enumember\", \"comparison\", \"cause\", \"exp-reason\", \"exp-evidence\", \"condition\", \"summary\", \"bg-compare\", \"elab-example\", \"elab-definition\", \"cause-result\", \"findings\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "bg-general \t \n",
+ "elab-addition \t \n",
+ "manner-means \t manner-means\n",
+ "attribution \t attribution\n",
+ "evaluation \t evaluation\n",
+ "enablement \t enablement\n",
+ "elab-aspect \t \n",
+ "joint \t \n",
+ "temporal \t \n",
+ "result \t cause\n",
+ "bg-goal \t \n",
+ "progression \t \n",
+ "contrast \t contrast\n",
+ "elab-process_step \t \n",
+ "elab-enumember \t \n",
+ "comparison \t comparison\n",
+ "cause \t cause\n",
+ "exp-reason \t explanation\n",
+ "exp-evidence \t explanation\n",
+ "condition \t condition\n",
+ "summary \t summary\n",
+ "bg-compare \t \n",
+ "elab-example \t elaboration\n",
+ "elab-definition \t elaboration\n",
+ "cause-result \t cause\n",
+ "findings \t \n"
+ ]
+ }
+ ],
+ "source": [
+ "for label in rst:\n",
+ " temp = ''\n",
+ " for k, v in subs.items():\n",
+ " if label in v:\n",
+ " temp = k.lower()\n",
+ " elif '-' in label:\n",
+ " for l in label.split('-'):\n",
+ " if l in v:\n",
+ " temp = temp = k.lower()\n",
+ " elif '.' in label:\n",
+ " for l in label.split('.'):\n",
+ " if l in v:\n",
+ " temp = temp = k.lower()\n",
+ " \n",
+ " print(label, '\\t', temp)\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/utils.py b/utils.py
index aa80eeabe4455324dd7e40ccf4045c97dc14258d..2802738e7da4ed2d3de635c08d8b6eb1a1e81eb2 100644
--- a/utils.py
+++ b/utils.py
@@ -213,13 +213,14 @@ def print_results_to_file(corpus,
test_sentences,
test_results,
inv_mappings_dict,
- substitutions_file):
+ substitutions_file,
+ output_folder):
''' Function to print a new file with the test predictions per
the specifications of the Shared task.
Returns: one file per corpus with predictions.
'''
- output_folder = 'results'
+# output_folder = 'results'
header = '\t'.join(['doc',
'unit1_toks',
'unit2_toks',
@@ -260,6 +261,8 @@ def print_results_to_file(corpus,
pass
temp = sent[:11] + [label]
results_to_write.append(temp)
+
+ assert len(results_to_write) == len(test_sentences)
with open(output_folder + '/' + corpus + '.tsv', 'a+', encoding='utf-8') as f:
f.write(header + '\n')