Skip to content
Snippets Groups Projects
Commit 30190c63 authored by emetheni's avatar emetheni
Browse files

zero-shot models

parent ddd2af48
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
see_results.ipynb deleted 100644 → 0
+ 0
− 411
View file @ ddd2af48
%% Cell type:code id: tags:
``` python
from sklearn.metrics import accuracy_score
import os, io
from collections import OrderedDict, Counter
```
%% Cell type:code id: tags:
``` python
connectives = {"elaboration": ["and", "also", "besides", "further", "furthermore", "too", "moreover", "in addition", "then", "of equal importance", "equally important", "another", "additionally", "also", "moreover", "furthermore", "again", "further", "then", "besides", "too", "similarly", "correspondingly", "indeed", "regarding"],
"time": ["next", "afterward", "finally", "later", "last", "lastly", "at last", "now", "subsequently", "then", "when", "soon", "thereafter", "after a short time", "the next week", "a minute later", "in the meantime", "meanwhile", "on the following day", "at length", "ultimately", "presently"],
"sequence": ["first", "second", "third", "finally", "hence", "next", "then", "from here on", "to begin with", "last of all", "after", "before", "as soon as", "in the end", "gradually", "when", "after", "after that", "afterwards", "next", "subsequently", "later (on)", "followed by", "to go on to", "finally", "another", "additionally", "finally moreover", "also", "subsequently", "eventually", "next", "then"],
"example": ["for example", "to illustrate", "for instance", "to be specific", "such as", "moreover", "furthermore", "just as important", "similarly", "in the same way", "for example", "for instance", "namely", "such as", "as follows", "as exemplified by", "such as", "including", "especially", "particularly", "in particular", "notably", "mainly"],
"result": ["as a result", "hence", "so", "accordingly", "as a consequence", "consequently", "thus", "since", "therefore", "for this reason", "because of this", "therefore", "accordingly", "as a result of", "the result is/results are", "the consequence is", "resulting from", "consequently", "it can be seen", "evidence illustrates that", "because of this", "thus", "hence", "for this reason", "owing to x", "this suggests that", "it follows that", "otherwise", "in that case", "that implies", "As a result", "therefore", "thus"],
"purpose": ["for this purpose", "with this in mind", "for this reason"],
"comparison": ["like", "in the same manner", "as so", "similarly"],
"contrast": ["but", "in contrast", "conversely", "however", "still", "nevertheless", "nonetheless", "yet", "and yet", "on the other hand", "on the contrary", "or", "in spite of this", "actually", "in fact", "whereas", "conversely", "in comparison", "by contrast", "in contrast", "contrasting", "alternatively", "although", "otherwise", "instead"],
"summary": ["in summary", "to sum up", "to repeat", "briefly", "in short", "finally", "on the whole", "therefore", "as I have said", "in conclusion", "as seen", "in conclusion", "therefore", "to conclude", "on the whole", "hence", "thus to summarise", "altogether", "overall"],
"rephrasing": ["in other terms", "rather", "or", "better", "in view of this", "in contrast"]}
```
%% Cell type:code id: tags:
``` python
def parse_data(infile, string_input=False) -> list:
"""
This function is to read a gold or a pred file to obtain the label column for accuracy calculation.
:param infile: shared task .rels file
:param string_input: If True, files are replaced by strings with file contents (for import inside other scripts)
:return: a list of labels
"""
if not string_input:
data = io.open(infile, encoding="utf-8").read().strip().replace("\r", "")
else:
data = infile.strip()
labels = [line.split("\t")[-1].lower()
for i, line in enumerate(data.split("\n")) if "\t" in line and i>0]
sentences = [(line.split("\t")[3], line.split("\t")[4], line.split("\t")[-3])
for i, line in enumerate(data.split("\n")) if "\t" in line and i>0]
return sentences, labels
def get_accuracy_score(gold_file, pred_file, string_input=False) -> dict:
_, gold_labels = parse_data(gold_file, string_input)
_, pred_labels = parse_data(pred_file, string_input)
filename = gold_file.split(os.sep)[-1]
assert len(gold_labels) == len(pred_labels), "FATAL: different number of labels detected in gold and pred"
acc = accuracy_score(gold_labels, pred_labels)
score_dict = {"filename": filename,
"acc_score": round(acc, 4),
"gold_rel_count": len(gold_labels),
"pred_rel_count": len(pred_labels)}
return score_dict
def separate_right_wrong(gold_file, pred_file, string_input=False):
rights = []
wrongs = []
gold_sents, gold_labels = parse_data(gold_file, string_input)
pred_sents, pred_labels = parse_data(pred_file, string_input)
for n in range(len(gold_sents)):
if gold_labels[n] == pred_labels[n]:
rights.append([gold_sents[n], gold_labels[n], pred_labels[n]])
else:
wrongs.append([gold_sents[n], gold_labels[n], pred_labels[n]])
return rights, wrongs
```
%% Cell type:code id: tags:
``` python
# Print accuracies
model = 'A_15-epochs_frozen-1_2'
corpus = 'eng.dep.covdtb'
gold_path = '/users/melodi/emetheni/clean_data/'
results_path = 'results/test/' + model + '/'
corpora = sorted([x[:-4] for x in os.listdir('results/test/' + model)
if not "DS" in x if not 'ipy' in x])
# for corpus in corpora:
# score = get_accuracy_score(gold_path + corpus + '/' + corpus + '_test.rels',
# results_path + corpus + '.tsv')
# print(corpus, '\t', score['acc_score'])
```
%% Cell type:code id: tags:
``` python
# Separate
# model = 'A_15-epochs_frozen-1_2'
# corpus = 'eng.dep.covdtb'
model = 'A_15-epochs_frozen-1-2-3_3'
corpus = 'eng.rst.gum'
gold_path = '/users/melodi/emetheni/clean_data/'
results_path = 'results/test/' + model + '/'
corpora = sorted([x[:-4] for x in os.listdir('results/test/' + model)
if not "DS" in x if not 'ipy' in x])
rights, wrongs = separate_right_wrong(gold_path + corpus + '/' + corpus + '_test.rels',
results_path + corpus + '.tsv')
rights_count = dict(OrderedDict(Counter([x[-1] for x in rights])))
wrongs_count = dict(OrderedDict(Counter([x[-1] for x in wrongs])))
# for label in sorted(set(list(rights_count.keys()) + list(wrongs_count.keys())), reverse=False):
# if label in rights_count:
# r = rights_count[label]
# else:
# r = 0
# if label in wrongs_count:
# w = wrongs_count[label]
# else:
# w = 0
# print(label, '\t', r, '\t', w)
```
%% Cell type:code id: tags:
``` python
# Presence of connectives in right/wrong sents
counter = 0
for sent in rights:
sentence = (sent[0][0] + ' ' + sent[0][1]).lower()
if sent[1] in connectives:
if any(x in sentence for x in connectives[sent[1]]):
# print(sent)
counter += 1
print('rights', counter, '/', len(rights), counter/len(rights))
counter = 0
for sent in wrongs:
sentence = (sent[0][0] + ' ' + sent[0][1]).lower()
if sent[1] in connectives:
if any(x in sentence for x in connectives[sent[1]]):
# print(sent)
counter += 1
print('wrongs', counter, '/', len(wrongs), counter/len(wrongs))
```
%% Output
rights 203 / 1657 0.12251056125528063
wrongs 71 / 918 0.07734204793028322
%% Cell type:code id: tags:
``` python
# See direction
counter = 0
for sent in rights:
if sent[0][2] == '1<2':
counter += 1
print('rights', counter, '/', len(rights), counter/len(rights))
counter = 0
for sent in wrongs:
if sent[0][2] == '1<2':
counter += 1
print('wrongs', counter, '/', len(wrongs), counter/len(wrongs))
```
%% Output
rights 1253 / 1657 0.756185878092939
wrongs 735 / 918 0.8006535947712419
%% Cell type:code id: tags:
``` python
rights[:10]
```
%% Output
[[('The prevalence of discrimination across racial groups in contemporary America :',
'The current study seeks to build on this research',
'1>2'),
'organization',
'organization'],
[('The prevalence of discrimination across racial groups in contemporary America :',
'Results from a nationally representative sample of adults',
'1<2'),
'elaboration',
'elaboration'],
[('Introduction .',
'The current study seeks to build on this research',
'1>2'),
'organization',
'organization'],
[('Personal experiences of discrimination and bias have been the focus of much social science research .',
'In many respects , researchers already possess a wealth of knowledge',
'1>2'),
'context',
'context'],
[('Personal experiences of discrimination and bias have been the focus of much social science research .',
'[ 1 - 3 ]',
'1<2'),
'explanation',
'explanation'],
[('Sociologists have explored the adverse consequences of discrimination',
'[ 3 – 5 ] ;',
'1<2'),
'explanation',
'explanation'],
[('Sociologists have explored the adverse consequences of discrimination',
'psychologists have examined the mental processes',
'1<2'),
'joint',
'joint'],
[('psychologists have examined the mental processes',
'that underpin conscious and unconscious biases',
'1<2'),
'elaboration',
'elaboration'],
[('psychologists have examined the mental processes', '[ 6 ] ;', '1<2'),
'explanation',
'explanation'],
[('Sociologists have explored the adverse consequences of discrimination',
'neuroscientists have examined the neurobiological underpinnings of discrimination',
'1<2'),
'joint',
'joint']]
%% Cell type:code id: tags:
``` python
subs = {"Attribution": ["attribution", "attribution-negative"],
"Background": ["background", "circumstance"],
"Cause": ["cause", "result", "consequence"],
"Comparison": ["comparison", "preference", "analogy", "proportion"],
"Condition": ["condition", "hypothetical", "contingency", "otherwise"],
"Contrast": ["contrast", "concession", "antithesis"],
"Elaboration": ["elaboration-additional", "elaboration-general-specific", "elaboration-part-whole", "elaboration-process-step", "elaboration-object-attribute", "elaboration-set-member", "example", "definition"],
"Enablement": ["purpose", "enablement"],
"Evaluation": ["evaluation", "interpretation", "conclusion", "comment"],
"Explanation": ["evidence", "explanation-argumentative", "reason"],
"Joint": ["list", "disjunction"],
"Manner-Means": ["manner", "means"],
"Topic-Comment": ["problem-solution", "question-answer", "statement-response", "topic-comment", "comment-topic", "rhetorical-question"],
"Summary": ["summary", "restatement"],
"Temporal": ["temporal-before", "temporal-after", "temporal-same-time", "sequence", "inverted-sequence"],
"Topic Change": ["topic-shift", "topic-drift"]}
```
%% Cell type:code id: tags:
``` python
rst = ["bg-general", "elab-addition", "manner-means", "attribution", "evaluation", "enablement", "elab-aspect", "joint", "temporal", "result", "bg-goal", "progression", "contrast", "elab-process_step", "elab-enumember", "comparison", "cause", "exp-reason", "exp-evidence", "condition", "summary", "bg-compare", "elab-example", "elab-definition", "cause-result", "findings"]
```
%% Cell type:code id: tags:
``` python
for label in rst:
temp = ''
for k, v in subs.items():
if label in v:
temp = k.lower()
elif '-' in label:
for l in label.split('-'):
if l in v:
temp = temp = k.lower()
elif '.' in label:
for l in label.split('.'):
if l in v:
temp = temp = k.lower()
print(label, '\t', temp)
```
%% Output
bg-general
elab-addition
manner-means manner-means
attribution attribution
evaluation evaluation
enablement enablement
elab-aspect
joint
temporal
result cause
bg-goal
progression
contrast contrast
elab-process_step
elab-enumember
comparison comparison
cause cause
exp-reason explanation
exp-evidence explanation
condition condition
summary summary
bg-compare
elab-example elaboration
elab-definition elaboration
cause-result cause
findings
%% Cell type:code id: tags:
``` python
```
utils.py
+ 18
− 7
View file @ 30190c63
......@@ -20,7 +20,14 @@ def open_mappings(mappings_file):
next(f)
for l in f:
mappings[l.split('\t')[0]] = int(l.strip().split('\t')[-1])
inv_mappings = {v:k for k, v in mappings.items()}
inv_mappings = {}
# for label, num in mappings.items():
# if not num in inv_mappings:
# inv_mappings[num] = label
# else:
# subs[label] = inv_mappings[num]
return mappings, inv_mappings
......@@ -158,7 +165,8 @@ def open_sentences(path_to_corpora, mappings_dict):
'''
corpora = [folder for folder in os.listdir(path_to_corpora)
if not any(i in folder for i in ['.md', 'DS_', 'utils', 'ipynb'])]
if not any(i in folder for i in ['.md', 'DS_', 'utils', 'ipynb'])
]
# ---------------------
train_sentences = []
......@@ -177,7 +185,9 @@ def open_sentences(path_to_corpora, mappings_dict):
# open normal files
train_file = ['/'.join([path_to_corpora, corpus, x])
for x in os.listdir(path_to_corpora + '/' + corpus)
if 'train' in x and 'rels' in x][0]
if 'train' in x and 'rels' in x
# attention! we block training for some languages if we want HERE
if not args.langs_to_use in x][0]
temp = open_file(train_file, mappings_dict)
# train_sentences += open_file_with_lang(train_file, mappings_dict)
train_sentences += temp
......@@ -207,7 +217,7 @@ def open_sentences(path_to_corpora, mappings_dict):
# test_dict_sentences[corpus] += open_file_with_lang(test_file, mappings_dict)
labels = {framework:set(all_labels[framework]) for framework in all_labels}
return train_sentences, dev_dict_sentences, test_dict_sentences, labels
......@@ -376,16 +386,17 @@ def get_better_predictions(model,
all_preds += [final_probs]
all_labels += batch_labels
# get the top predictions in order to get the acc
top_preds = []
for probs in all_preds:
top_preds.append(max(zip(probs.values(), probs.keys()))[1])
test_acc = round(accuracy_score(labels, top_preds), 4)
test_acc = round(accuracy_score(all_labels, top_preds), 4)
if print_results:
print('better:', '\t', test_acc)
print('better:', '\t', test_acc, flush='')
return all_labels, all_preds
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment