Training Loss looks wild. Please Help

bert_no_ernie.py

@@ -89,12 +89,13 @@ def training_loop(model,criterion,optimizer,train_loader):
             input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
             # Feed Model with Data
             outputs = model(input_ids, attention_mask=att_mask)
-            print(f"Output Tensor: {outputs}")
+            # print(f"Output Tensor: {outputs}")
             loss = criterion(outputs,labels.float())
             loss.backward()
             optimizer.step()
             total_loss+=loss.item()
         print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")  
+        return (total_loss/len(train_loader))       
 
 def eval_loop(model,criterion,validation_loader):
         model.eval()
@@ -124,13 +125,13 @@ def generate_tokens(tokenizer,raw_data):
             )
 
 if __name__ == "__main__":
-    
+    torch.manual_seed(501)
     # Initialize Bert Model with dropout probability and Num End Layers
     mybert = CustomBert(0.1)
     print("Bert Initialized")
     
     # Set Max Epoch Amount
-    EPOCH = 2
+    EPOCH = 50
    
     # Read Raw Data from csv and save as DataFrame
     df = pd.read_csv("./data/hack.csv",encoding="latin1")
@@ -160,14 +161,22 @@ if __name__ == "__main__":
     print("DataLoaders created")
 
     # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
-    criterion_bce = nn.CrossEntropyLoss()
-    optimizer_adam = optim.Adam(mybert.parameters(), lr = 1e-5)
+    criterion_bce = nn.BCELoss()
+    optimizer_adam = optim.Adam(mybert.parameters(), lr = 3e-5)
 
     # Set Scheduler for dynamically Learning Rate adjustment
-    # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
-    
+    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
+    loss_values = []
     for epoch in range(EPOCH):
         print(f"For {epoch+1} the Scores are: ")
-        training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader)
+        loss_values.append(training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader))
         # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader)
-        # scheduler.step()
+        scheduler.step(.1)
+    
+    # Visualize Training Loss 
+    plt.plot(loss_values)
+    plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
+    plt.title("Training Loss")
+    plt.xlabel("Num Epochs")
+    plt.ylabel("Total Loss of Epoch")
+    plt.show()