Add logs

Linker/Linker.py

 import os
 import sys
-from datetime import datetime
+import datetime
 
+import time
+
+import torch
 import torch.nn.functional as F
+from torch import Module
 from torch.nn import Sequential, LayerNorm, Dropout
 from torch.optim import AdamW
 from torch.utils.data import TensorDataset, random_split
 from torch.utils.tensorboard import SummaryWriter
+from tqdm import tqdm
 from transformers import get_cosine_schedule_with_warmup
 
 from Configuration import Configuration
@@ -21,6 +26,15 @@ from Linker.utils_linker import find_pos_neg_idexes, get_atoms_batch, FFN, get_a
 from Supertagger import *
 from utils import pad_sequence
 
+def format_time(elapsed):
+    '''
+    Takes a time in seconds and returns a string hh:mm:ss
+    '''
+    # Round to the nearest second.
+    elapsed_rounded = int(round(elapsed))
+
+    # Format as hh:mm:ss
+    return str(datetime.timedelta(seconds=elapsed_rounded))
 
 def output_create_dir():
     """
@@ -184,20 +198,24 @@ class Linker(Module):
             checkpoint_dir, writer = output_create_dir()
 
         for epoch_i in range(epochs):
-            avg_train_loss, avg_accuracy_train = self.train_epoch(training_dataloader)
+            print("")
+            print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
+            print('Training...')
+            avg_train_loss, avg_accuracy_train, training_time = self.train_epoch(training_dataloader)
 
-            print("Average Loss on train dataset : ", avg_train_loss)
-            print("Average Accuracy on train dataset : ", avg_accuracy_train)
+
+            print("")
+            print(f'Epoch: {epoch_i + 1:02} | Epoch Time: {training_time}')
+            print(f'\tTrain Loss: {avg_train_loss:.3f} | Train Acc: {avg_accuracy_train * 100:.2f}%')
+            if validation_rate > 0.0:
+                with torch.no_grad():
+                    loss_test, accuracy_test = self.eval_epoch(validation_dataloader, self.cross_entropy_loss)
+                print(f'\tVal Loss: {loss_test:.3f} | Val Acc: {accuracy_test * 100:.2f}%')
 
             if checkpoint:
                 self.__checkpoint_save(
                     path=os.path.join("Output", 'linker' + datetime.today().strftime('%d-%m_%H-%M') + '.pt'))
 
-            if validation_rate > 0.0:
-                with torch.no_grad():
-                    loss_test, accuracy_test = self.eval_epoch(validation_dataloader, self.cross_entropy_loss)
-                    print("Average Loss on test dataset : ", loss_test)
-                    print("Average Accuracy on test dataset : ", accuracy_test)
 
             if tensorboard:
                 writer.add_scalars(f'Accuracy', {
@@ -226,44 +244,48 @@ class Linker(Module):
         # Reset the total loss for this epoch.
         epoch_loss = 0
         accuracy_train = 0
+        t0 = time.time()
 
         self.train()
 
         # For each batch of training data...
-        for step, batch in enumerate(training_dataloader):
-            # Unpack this training batch from our dataloader
-            batch_atoms = batch[0].to(self.device)
-            batch_polarity = batch[1].to(self.device)
-            batch_true_links = batch[2].to(self.device)
-            batch_sentences_tokens = batch[3].to(self.device)
-            batch_sentences_mask = batch[4].to(self.device)
+        with tqdm(training_dataloader, unit="batch") as tepoch:
+            for batch in tepoch:
+                # Unpack this training batch from our dataloader
+                batch_atoms = batch[0].to(self.device)
+                batch_polarity = batch[1].to(self.device)
+                batch_true_links = batch[2].to(self.device)
+                batch_sentences_tokens = batch[3].to(self.device)
+                batch_sentences_mask = batch[4].to(self.device)
 
-            self.optimizer.zero_grad()
+                self.optimizer.zero_grad()
 
-            # get sentence embedding from BERT which is already trained
-            logits, sentences_embedding = self.Supertagger.forward(batch_sentences_tokens, batch_sentences_mask)
+                # get sentence embedding from BERT which is already trained
+                logits, sentences_embedding = self.Supertagger.forward(batch_sentences_tokens, batch_sentences_mask)
 
-            # Run the kinker on the categories predictions
-            logits_predictions = self(batch_atoms, batch_polarity, sentences_embedding, batch_sentences_mask)
+                # Run the kinker on the categories predictions
+                logits_predictions = self(batch_atoms, batch_polarity, sentences_embedding, batch_sentences_mask)
 
-            linker_loss = self.cross_entropy_loss(logits_predictions, batch_true_links)
-            # Perform a backward pass to calculate the gradients.
-            epoch_loss += float(linker_loss)
-            linker_loss.backward()
+                linker_loss = self.cross_entropy_loss(logits_predictions, batch_true_links)
+                # Perform a backward pass to calculate the gradients.
+                epoch_loss += float(linker_loss)
+                linker_loss.backward()
 
-            # This is to help prevent the "exploding gradients" problem.
-            # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
+                # This is to help prevent the "exploding gradients" problem.
+                # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
 
-            # Update parameters and take a step using the computed gradient.
-            self.optimizer.step()
+                # Update parameters and take a step using the computed gradient.
+                self.optimizer.step()
 
-            pred_axiom_links = torch.argmax(logits_predictions, dim=3)
-            accuracy_train += mesure_accuracy(batch_true_links, pred_axiom_links)
+                pred_axiom_links = torch.argmax(logits_predictions, dim=3)
+                accuracy_train += mesure_accuracy(batch_true_links, pred_axiom_links)
 
+        # Measure how long this epoch took.
+        training_time = format_time(time.time() - t0)
         avg_train_loss = epoch_loss / len(training_dataloader)
         avg_accuracy_train = accuracy_train / len(training_dataloader)
 
-        return avg_train_loss, avg_accuracy_train
+        return avg_train_loss, avg_accuracy_train, training_time
 
     def predict(self, categories, sents_embedding, sents_mask=None):
         r"""Prediction from categories output by BERT and hidden_state from BERT