From d160eeee66126d1617be729cc4363cfddedd5b18 Mon Sep 17 00:00:00 2001
From: Caroline DE POURTALES <cdepourt@montana.irit.fr>
Date: Thu, 30 Jun 2022 14:01:10 +0200
Subject: [PATCH] best score 81%
---
Configuration/config.ini | 8 ++++----
Linker/Linker.py | 13 ++++++++-----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/Configuration/config.ini b/Configuration/config.ini
index f1e5c0d..0cf354c 100644
--- a/Configuration/config.ini
+++ b/Configuration/config.ini
@@ -14,11 +14,11 @@ dim_encoder = 768
[MODEL_LINKER]
nhead=8
dim_emb_atom = 256
-dim_feedforward_transformer = 768
+dim_feedforward_transformer = 512
num_layers=3
-dim_cat_inter=512
-dim_cat_out=256
-dim_intermediate_FFN=128
+dim_cat_inter=768
+dim_cat_out=512
+dim_intermediate_FFN=256
dim_pre_sinkhorn_transfo=32
dropout=0.1
sinkhorn_iters=5
diff --git a/Linker/Linker.py b/Linker/Linker.py
index 9d44b54..370b25b 100644
--- a/Linker/Linker.py
+++ b/Linker/Linker.py
@@ -100,9 +100,9 @@ class Linker(Module):
self.sub_atoms_type_list = list(atom_map_redux.keys())
self.atom_encoder = Embedding(atom_vocab_size, self.dim_emb_atom, padding_idx=self.padding_id)
self.atom_encoder.weight.data.uniform_(-0.1, 0.1)
- self.position_encoder = PositionalEncoding(self.dim_emb_atom, 0.1, max_len=self.max_atoms_in_sentence)
+ self.position_encoder = PositionalEncoding(self.dim_emb_atom, dropout, max_len=self.max_atoms_in_sentence)
encoder_layer = TransformerEncoderLayer(d_model=self.dim_emb_atom, nhead=self.nhead,
- dim_feedforward=self.dim_feedforward_transformer, dropout=0.1)
+ dim_feedforward=self.dim_feedforward_transformer, dropout=dropout)
self.transformer = TransformerEncoder(encoder_layer, num_layers=self.num_layers)
# Concatenation with word embedding
@@ -116,11 +116,11 @@ class Linker(Module):
# Division into positive and negative
self.pos_transformation = Sequential(
- FFN(self.dim_cat_out, dim_intermediate_FFN, 0.1, d_out=dim_pre_sinkhorn_transfo),
+ FFN(self.dim_cat_out, dim_intermediate_FFN, dropout, d_out=dim_pre_sinkhorn_transfo),
LayerNorm(dim_pre_sinkhorn_transfo, eps=1e-8)
)
self.neg_transformation = Sequential(
- FFN(self.dim_cat_out, dim_intermediate_FFN, 0.1, d_out=dim_pre_sinkhorn_transfo),
+ FFN(self.dim_cat_out, dim_intermediate_FFN, dropout, d_out=dim_pre_sinkhorn_transfo),
LayerNorm(dim_pre_sinkhorn_transfo, eps=1e-8)
)
@@ -128,7 +128,7 @@ class Linker(Module):
self.cross_entropy_loss = SinkhornLoss()
self.optimizer = AdamW(self.parameters(),
lr=learning_rate)
- self.scheduler = StepLR(self.optimizer, step_size=2, gamma=0.5)
+ self.scheduler = StepLR(self.optimizer, step_size=3, gamma=0.5)
self.to(self.device)
@@ -257,6 +257,8 @@ class Linker(Module):
if tensorboard:
writer.add_scalars(f'Accuracy', {
'Train': avg_accuracy_train}, epoch_i)
+ writer.add_scalars(f'Learning rate', {
+ 'learning_rate': self.scheduler.get_last_lr()}, epoch_i)
writer.add_scalars(f'Loss', {
'Train': avg_train_loss}, epoch_i)
if validation_rate > 0.0:
@@ -319,6 +321,7 @@ class Linker(Module):
accuracy_train += mesure_accuracy(batch_true_links, pred_axiom_links, self.max_atoms_in_one_type)
self.scheduler.step()
+ print("learning rate ", self.scheduler.get_last_lr())
# Measure how long this epoch took.
training_time = format_time(time.time() - t0)
--
GitLab