Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
DeepGrail Linker
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
PNRIA
Global Helper
DeepGrail Linker
Commits
ff8dc3af
Commit
ff8dc3af
authored
3 years ago
by
Julien Rabault
Browse files
Options
Downloads
Patches
Plain Diff
Add logs
parent
e8d49aae
Branches
Branches containing commit
No related tags found
2 merge requests
!6
Linker with transformer
,
!5
Linker with transformer
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
Linker/Linker.py
+54
-32
54 additions, 32 deletions
Linker/Linker.py
with
54 additions
and
32 deletions
Linker/Linker.py
+
54
−
32
View file @
ff8dc3af
import
os
import
sys
from
datetime
import
datetime
import
datetime
import
time
import
torch
import
torch.nn.functional
as
F
from
torch
import
Module
from
torch.nn
import
Sequential
,
LayerNorm
,
Dropout
from
torch.optim
import
AdamW
from
torch.utils.data
import
TensorDataset
,
random_split
from
torch.utils.tensorboard
import
SummaryWriter
from
tqdm
import
tqdm
from
transformers
import
get_cosine_schedule_with_warmup
from
Configuration
import
Configuration
...
...
@@ -21,6 +26,15 @@ from Linker.utils_linker import find_pos_neg_idexes, get_atoms_batch, FFN, get_a
from
Supertagger
import
*
from
utils
import
pad_sequence
def
format_time
(
elapsed
):
'''
Takes a time in seconds and returns a string hh:mm:ss
'''
# Round to the nearest second.
elapsed_rounded
=
int
(
round
(
elapsed
))
# Format as hh:mm:ss
return
str
(
datetime
.
timedelta
(
seconds
=
elapsed_rounded
))
def
output_create_dir
():
"""
...
...
@@ -184,20 +198,24 @@ class Linker(Module):
checkpoint_dir
,
writer
=
output_create_dir
()
for
epoch_i
in
range
(
epochs
):
avg_train_loss
,
avg_accuracy_train
=
self
.
train_epoch
(
training_dataloader
)
print
(
""
)
print
(
'
======== Epoch {:} / {:} ========
'
.
format
(
epoch_i
+
1
,
epochs
))
print
(
'
Training...
'
)
avg_train_loss
,
avg_accuracy_train
,
training_time
=
self
.
train_epoch
(
training_dataloader
)
print
(
"
Average Loss on train dataset :
"
,
avg_train_loss
)
print
(
"
Average Accuracy on train dataset :
"
,
avg_accuracy_train
)
print
(
""
)
print
(
f
'
Epoch:
{
epoch_i
+
1
:
02
}
| Epoch Time:
{
training_time
}
'
)
print
(
f
'
\t
Train Loss:
{
avg_train_loss
:
.
3
f
}
| Train Acc:
{
avg_accuracy_train
*
100
:
.
2
f
}
%
'
)
if
validation_rate
>
0.0
:
with
torch
.
no_grad
():
loss_test
,
accuracy_test
=
self
.
eval_epoch
(
validation_dataloader
,
self
.
cross_entropy_loss
)
print
(
f
'
\t
Val Loss:
{
loss_test
:
.
3
f
}
| Val Acc:
{
accuracy_test
*
100
:
.
2
f
}
%
'
)
if
checkpoint
:
self
.
__checkpoint_save
(
path
=
os
.
path
.
join
(
"
Output
"
,
'
linker
'
+
datetime
.
today
().
strftime
(
'
%d-%m_%H-%M
'
)
+
'
.pt
'
))
if
validation_rate
>
0.0
:
with
torch
.
no_grad
():
loss_test
,
accuracy_test
=
self
.
eval_epoch
(
validation_dataloader
,
self
.
cross_entropy_loss
)
print
(
"
Average Loss on test dataset :
"
,
loss_test
)
print
(
"
Average Accuracy on test dataset :
"
,
accuracy_test
)
if
tensorboard
:
writer
.
add_scalars
(
f
'
Accuracy
'
,
{
...
...
@@ -226,44 +244,48 @@ class Linker(Module):
# Reset the total loss for this epoch.
epoch_loss
=
0
accuracy_train
=
0
t0
=
time
.
time
()
self
.
train
()
# For each batch of training data...
for
step
,
batch
in
enumerate
(
training_dataloader
):
# Unpack this training batch from our dataloader
batch_atoms
=
batch
[
0
].
to
(
self
.
device
)
batch_polarity
=
batch
[
1
].
to
(
self
.
device
)
batch_true_links
=
batch
[
2
].
to
(
self
.
device
)
batch_sentences_tokens
=
batch
[
3
].
to
(
self
.
device
)
batch_sentences_mask
=
batch
[
4
].
to
(
self
.
device
)
with
tqdm
(
training_dataloader
,
unit
=
"
batch
"
)
as
tepoch
:
for
batch
in
tepoch
:
# Unpack this training batch from our dataloader
batch_atoms
=
batch
[
0
].
to
(
self
.
device
)
batch_polarity
=
batch
[
1
].
to
(
self
.
device
)
batch_true_links
=
batch
[
2
].
to
(
self
.
device
)
batch_sentences_tokens
=
batch
[
3
].
to
(
self
.
device
)
batch_sentences_mask
=
batch
[
4
].
to
(
self
.
device
)
self
.
optimizer
.
zero_grad
()
self
.
optimizer
.
zero_grad
()
# get sentence embedding from BERT which is already trained
logits
,
sentences_embedding
=
self
.
Supertagger
.
forward
(
batch_sentences_tokens
,
batch_sentences_mask
)
# get sentence embedding from BERT which is already trained
logits
,
sentences_embedding
=
self
.
Supertagger
.
forward
(
batch_sentences_tokens
,
batch_sentences_mask
)
# Run the kinker on the categories predictions
logits_predictions
=
self
(
batch_atoms
,
batch_polarity
,
sentences_embedding
,
batch_sentences_mask
)
# Run the kinker on the categories predictions
logits_predictions
=
self
(
batch_atoms
,
batch_polarity
,
sentences_embedding
,
batch_sentences_mask
)
linker_loss
=
self
.
cross_entropy_loss
(
logits_predictions
,
batch_true_links
)
# Perform a backward pass to calculate the gradients.
epoch_loss
+=
float
(
linker_loss
)
linker_loss
.
backward
()
linker_loss
=
self
.
cross_entropy_loss
(
logits_predictions
,
batch_true_links
)
# Perform a backward pass to calculate the gradients.
epoch_loss
+=
float
(
linker_loss
)
linker_loss
.
backward
()
# This is to help prevent the "exploding gradients" problem.
# torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
# This is to help prevent the "exploding gradients" problem.
# torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
# Update parameters and take a step using the computed gradient.
self
.
optimizer
.
step
()
# Update parameters and take a step using the computed gradient.
self
.
optimizer
.
step
()
pred_axiom_links
=
torch
.
argmax
(
logits_predictions
,
dim
=
3
)
accuracy_train
+=
mesure_accuracy
(
batch_true_links
,
pred_axiom_links
)
pred_axiom_links
=
torch
.
argmax
(
logits_predictions
,
dim
=
3
)
accuracy_train
+=
mesure_accuracy
(
batch_true_links
,
pred_axiom_links
)
# Measure how long this epoch took.
training_time
=
format_time
(
time
.
time
()
-
t0
)
avg_train_loss
=
epoch_loss
/
len
(
training_dataloader
)
avg_accuracy_train
=
accuracy_train
/
len
(
training_dataloader
)
return
avg_train_loss
,
avg_accuracy_train
return
avg_train_loss
,
avg_accuracy_train
,
training_time
def
predict
(
self
,
categories
,
sents_embedding
,
sents_mask
=
None
):
r
"""
Prediction from categories output by BERT and hidden_state from BERT
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment