Skip to content
Snippets Groups Projects
Commit ff8dc3af authored by Julien Rabault's avatar Julien Rabault
Browse files

Add logs

parent e8d49aae
Branches
No related tags found
2 merge requests!6Linker with transformer,!5Linker with transformer
import os
import sys
from datetime import datetime
import datetime
import time
import torch
import torch.nn.functional as F
from torch import Module
from torch.nn import Sequential, LayerNorm, Dropout
from torch.optim import AdamW
from torch.utils.data import TensorDataset, random_split
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
from transformers import get_cosine_schedule_with_warmup
from Configuration import Configuration
......@@ -21,6 +26,15 @@ from Linker.utils_linker import find_pos_neg_idexes, get_atoms_batch, FFN, get_a
from Supertagger import *
from utils import pad_sequence
def format_time(elapsed):
'''
Takes a time in seconds and returns a string hh:mm:ss
'''
# Round to the nearest second.
elapsed_rounded = int(round(elapsed))
# Format as hh:mm:ss
return str(datetime.timedelta(seconds=elapsed_rounded))
def output_create_dir():
"""
......@@ -184,20 +198,24 @@ class Linker(Module):
checkpoint_dir, writer = output_create_dir()
for epoch_i in range(epochs):
avg_train_loss, avg_accuracy_train = self.train_epoch(training_dataloader)
print("")
print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
avg_train_loss, avg_accuracy_train, training_time = self.train_epoch(training_dataloader)
print("Average Loss on train dataset : ", avg_train_loss)
print("Average Accuracy on train dataset : ", avg_accuracy_train)
print("")
print(f'Epoch: {epoch_i + 1:02} | Epoch Time: {training_time}')
print(f'\tTrain Loss: {avg_train_loss:.3f} | Train Acc: {avg_accuracy_train * 100:.2f}%')
if validation_rate > 0.0:
with torch.no_grad():
loss_test, accuracy_test = self.eval_epoch(validation_dataloader, self.cross_entropy_loss)
print(f'\tVal Loss: {loss_test:.3f} | Val Acc: {accuracy_test * 100:.2f}%')
if checkpoint:
self.__checkpoint_save(
path=os.path.join("Output", 'linker' + datetime.today().strftime('%d-%m_%H-%M') + '.pt'))
if validation_rate > 0.0:
with torch.no_grad():
loss_test, accuracy_test = self.eval_epoch(validation_dataloader, self.cross_entropy_loss)
print("Average Loss on test dataset : ", loss_test)
print("Average Accuracy on test dataset : ", accuracy_test)
if tensorboard:
writer.add_scalars(f'Accuracy', {
......@@ -226,44 +244,48 @@ class Linker(Module):
# Reset the total loss for this epoch.
epoch_loss = 0
accuracy_train = 0
t0 = time.time()
self.train()
# For each batch of training data...
for step, batch in enumerate(training_dataloader):
# Unpack this training batch from our dataloader
batch_atoms = batch[0].to(self.device)
batch_polarity = batch[1].to(self.device)
batch_true_links = batch[2].to(self.device)
batch_sentences_tokens = batch[3].to(self.device)
batch_sentences_mask = batch[4].to(self.device)
with tqdm(training_dataloader, unit="batch") as tepoch:
for batch in tepoch:
# Unpack this training batch from our dataloader
batch_atoms = batch[0].to(self.device)
batch_polarity = batch[1].to(self.device)
batch_true_links = batch[2].to(self.device)
batch_sentences_tokens = batch[3].to(self.device)
batch_sentences_mask = batch[4].to(self.device)
self.optimizer.zero_grad()
self.optimizer.zero_grad()
# get sentence embedding from BERT which is already trained
logits, sentences_embedding = self.Supertagger.forward(batch_sentences_tokens, batch_sentences_mask)
# get sentence embedding from BERT which is already trained
logits, sentences_embedding = self.Supertagger.forward(batch_sentences_tokens, batch_sentences_mask)
# Run the kinker on the categories predictions
logits_predictions = self(batch_atoms, batch_polarity, sentences_embedding, batch_sentences_mask)
# Run the kinker on the categories predictions
logits_predictions = self(batch_atoms, batch_polarity, sentences_embedding, batch_sentences_mask)
linker_loss = self.cross_entropy_loss(logits_predictions, batch_true_links)
# Perform a backward pass to calculate the gradients.
epoch_loss += float(linker_loss)
linker_loss.backward()
linker_loss = self.cross_entropy_loss(logits_predictions, batch_true_links)
# Perform a backward pass to calculate the gradients.
epoch_loss += float(linker_loss)
linker_loss.backward()
# This is to help prevent the "exploding gradients" problem.
# torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
# This is to help prevent the "exploding gradients" problem.
# torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
# Update parameters and take a step using the computed gradient.
self.optimizer.step()
# Update parameters and take a step using the computed gradient.
self.optimizer.step()
pred_axiom_links = torch.argmax(logits_predictions, dim=3)
accuracy_train += mesure_accuracy(batch_true_links, pred_axiom_links)
pred_axiom_links = torch.argmax(logits_predictions, dim=3)
accuracy_train += mesure_accuracy(batch_true_links, pred_axiom_links)
# Measure how long this epoch took.
training_time = format_time(time.time() - t0)
avg_train_loss = epoch_loss / len(training_dataloader)
avg_accuracy_train = accuracy_train / len(training_dataloader)
return avg_train_loss, avg_accuracy_train
return avg_train_loss, avg_accuracy_train, training_time
def predict(self, categories, sents_embedding, sents_mask=None):
r"""Prediction from categories output by BERT and hidden_state from BERT
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment