removed Main and transfered to notebook

bert_no_ernie.py

@@ -61,8 +61,10 @@ class CustomBert(nn.Module):
         # self.sm = nn.Softmax(dim=1)
          
     def forward(self, input_ids, attention_mask):
-        seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
+        x = self.bfsc(input_ids, attention_mask = attention_mask)
-        return self.classifier(self.dropout(seq_out[0]))
+        x = self.dropout(x[0])
+        x = self.classifier(x)
+        return x
         
     
     def freeze_bert_params(self):
@@ -73,21 +75,22 @@ class CustomBert(nn.Module):
         for param in self.bfsc.named_parameters():
             param[1].requires_grad_(True)
     
-def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool):
+def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool=False):
     model.train()
     if freeze_bert:
         model.freeze_bert_params()
     
     total_loss = 0
     len_train_loader = len(train_loader)
-    for index,train_batch in enumerate(train_loader):
+    for train_batch in train_loader:
+        
         # Set Gradient to Zero
         optimizer.zero_grad()
+       
         # Unpack batch values and "push" it to GPU
         input_ids, att_mask, labels = train_batch.values()
-        # print(f"{input_ids.shape}, {att_mask.shape}, {labels.shape}")
-        # print(f"Iteration {index} of {len_train_loader}")
         input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
+       
         # Feed Model with Data
         outputs = model(input_ids, attention_mask=att_mask)
         # print(f"{model.bfsc.}")
@@ -96,6 +99,7 @@ def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim
         loss.backward()
         optimizer.step()
         total_loss+=loss.item()
+
     print(f"Training Loss is {(total_loss/len(train_loader)):.4f}")  
     return (total_loss/len(train_loader))       
 
@@ -103,109 +107,47 @@ def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:D
     model.eval()
     total, correct = 0.0, 0.0
     total_loss = 0.0
-    best_loss = 10.0
+    best_loss = float("Inf")
     with torch.no_grad():
         for val_batch in validation_loader:
+
             input_ids, att_mask ,labels = val_batch.values()
             input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE)
+            
             outputs = model(input_ids,attention_mask=att_mask)
+            
             loss = criterion(outputs,labels)
             total_loss += loss.item()
+            
             predictions = torch.argmax(outputs,1)
             total += labels.size(0)
             correct += (predictions == labels).sum().item()
+            
             if total_loss/len(validation_loader) < best_loss:
                 best_loss = total_loss/len(validation_loader)
-                torch.save(model,"best_bert_model")
+                torch.save(model,"best_bert_model.pt")
-    print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Test Accuracy {correct/total*100:.4f}%")
+
+    print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Validation Accuracy {correct/total*100:.4f}%")
     return total_loss/len(validation_loader)
 
-def test_loop(model:CustomBert, criterion:nn.CrossEntropyLoss, test_loader:DataLoader):
+def test_loop(model:CustomBert, test_loader:DataLoader):
     for batch in test_loader:
         input_ids, att_mask, labels = batch.values()
         input_ids, att_mask, labels = input_ids.to(DEVICE), att_mask.to(DEVICE), labels.to(DEVICE)
         with torch.no_grad():
+            model = torch.load("best_bert_model")
+            model.to(DEVICE)
             output = model(input_ids,att_mask)
             output.detach().cpu().numpy()
             labels.detach().cpu().numpy()
             pred_flat = np.argmax(output,1).flatten()
             print(accuracy_score(labels,pred_flat))
 
-def performance_metrics(true_labels,predictions):
+def plot_metrics_loss_n_acc(train_loss,validation_loss,train_acc,validation_acc):
-    confusion_matrix(true_labels,predictions)
+    """
-    accuracy_score(true_labels,predictions)
+    Method that plots Loss and Accuracy of Training and Validation Data used in given modelinstance
-    f1_score(true_labels,predictions)
+    """
-    pass
+        # Visualize Training Loss 
-
-
-if __name__ == "__main__":
-    
-    # HYPERPARAMETERS
-    # Set Max Epoch Amount
-    EPOCH = 10
-    # DROPOUT-PROBABILITY
-    DROPOUT = 0.1
-    # BATCHSIZE
-    BATCH_SIZE = 16
-    #LEARNING RATE
-    LEARNING_RATE = 1e-5
-    # RANDOM SEED
-    RNDM_SEED = 501
-
-    torch.manual_seed(RNDM_SEED)
-    np.random.seed(RNDM_SEED)
-    torch.cuda.seed_all(RNDM_SEED)
-
-    # Initialize Bert Model with dropout probability and Num End Layers
-    mybert = CustomBert(DROPOUT)
-    print("Bert Initialized")
-    mybert.to(DEVICE)
-
-
-    # Read Raw Data from csv and save as DataFrame
-    df = pd.read_csv("./data/hack.csv",encoding="latin1")
-    print("Raw Data read")
-
-    # Initialize BertTokenizer from Pretrained 
-    tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
-    print("Tokenizer Initialized")
-
-    #Split DataFrame into Train and Test Sets
-    train,test = train_test_split(df,random_state=501,test_size=.2)
-    print("Splitted Data in Train and Test Sets")
-    test,val = train_test_split(test,random_state=501,test_size=.5)
-
-    # val = []
-    # Create Custom Datasets for Train and Test
-    train_data = SimpleHumorDataset(tokenizer,train)
-    val_data = SimpleHumorDataset(tokenizer,val)
-    test_data = SimpleHumorDataset(tokenizer,test)
-    print("Custom Datasets created") 
-
-
-   # Initialize Dataloader with Train and Test Sets
-    train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
-    validation_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
-    test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
-    print("DataLoaders created")
-
-    # Set criterion to Cross Entropy and define Adam Optimizer with model parameters and learning rate
-    criterion_cross_entropy = nn.CrossEntropyLoss()
-    optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
-    import time
-    # Set Scheduler for dynamically Learning Rate adjustment
-    loss_values = np.zeros(EPOCH)
-    eval_values = np.zeros(EPOCH)
-    freeze = False
-    
-    for epoch in range(EPOCH):
-        start = time.time()
-        print(f"For {epoch+1} the Scores are: ")
-        loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=freeze)
-        eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
-        end = time.time()
-        print((end-start),"seconds per epoch needed")
-    # Visualize Training Loss 
     # plt.plot(loss_values)
     # plt.plot(eval_values)
     # plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
@@ -214,5 +156,111 @@ if __name__ == "__main__":
     # plt.xlabel("Num Epochs")
     # plt.ylabel("Total Loss of Epoch")
     # plt.show()
-    for epoch in range(EPOCH):
+    pass
-        test_loop(mybert,criterion_cross_entropy,validation_loader)
+
+def plot_test_metrics(accuracy):
+    """
+    Plot Test Metrics of Model (Confiuson Matrix, Accuracy)
+    """
+    plt.plot(accuracy)
+    plt.hlines(np.mean(accuracy),0,len(accuracy),'red','dotted','Mean Accuracy %d'.format(np.mean(accuracy)))
+    plt.title("Accuracy of Test")
+    plt.xlabel("Num Epochs")
+    plt.ylabel("Accurcy 0.0 - 1.0")
+    plt.grid(True)
+    plt.legend()
+    plt.show()
+
+# def performance_metrics(true_labels,predictions):
+#     confusion_matrix(true_labels,predictions)
+#     accuracy_score(true_labels,predictions)
+#     f1_score(true_labels,predictions)
+#     pass
+
+def create_datasets(tokenizer:AutoTokenizer,dataframe:pd.DataFrame,train_split_ratio:float,val:bool=False)->tuple[SimpleHumorDataset,SimpleHumorDataset,SimpleHumorDataset]|tuple[SimpleHumorDataset,SimpleHumorDataset]:
+    if train_split_ratio > 1.0:
+        raise AssertionError("Trainsplit sollte kleiner(-gleich) 1.0 sein")
+    train,test = train_test_split(dataframe,train_size=train_split_ratio,random_state=501)
+    if val:
+        test,validation = train_test_split(test,train_size=.5,random_state=501)
+        return SimpleHumorDataset(tokenizer,train), SimpleHumorDataset(tokenizer,test), SimpleHumorDataset(tokenizer,validation)
+    return SimpleHumorDataset(tokenizer,train), SimpleHumorDataset(tokenizer,test)    
+
+def create_dataloaders(datasets:tuple|list,batchsize:int,shufflelist:list):
+    train_loader = DataLoader(datasets[0],batchsize,shuffle=shufflelist[0])
+    test_loader = DataLoader(datasets[1],batchsize,shuffle=shufflelist[1])
+    if len(datasets) == 3:
+        return train_loader, test_loader, DataLoader(datasets[2],batchsize,shuffle=shufflelist[2])
+    return train_loader, test_loader
+    
+
+# if __name__ == "__main__":
+    
+    # # HYPERPARAMETERS
+    # # Set Max Epoch Amount
+    # EPOCH = 10
+    # # DROPOUT-PROBABILITY
+    # DROPOUT = 0.1
+    # # BATCHSIZE
+    # BATCH_SIZE = 16
+    # #LEARNING RATE
+    # LEARNING_RATE = 1e-5
+    # # RANDOM SEED
+    # RNDM_SEED = 501
+    # # FREEZE Bert Layers
+    # FREEZE = True
+
+    # torch.manual_seed(RNDM_SEED)
+    # np.random.seed(RNDM_SEED)
+    # torch.cuda.manual_seed_all(RNDM_SEED)
+
+
+    # Initialize Bert Model with dropout probability and port to DEVICE
+    # mybert = CustomBert(DROPOUT)
+    # print("Bert Initialized")
+    # mybert.to(DEVICE)
+
+
+    # Read Raw Data from csv and save as DataFrame
+    # df = pd.read_csv("./data/hack.csv",encoding="latin1")
+    # print("Raw Data read")
+
+
+    # Initialize BertTokenizer from Pretrained 
+    # tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
+    # print("Tokenizer Initialized")
+
+
+    # Split DataFrame into Train and Test Sets
+    # Create Custom Datasets for Train and Test
+    # train_data,test_data,validation_data = create_datasets(tokenizer,df,.7,True)
+    # print("Splitted Data in Train and Test Sets")
+    # print("Custom Datasets created") 
+
+
+   # Initialize Dataloader with Train and Test Sets
+    # train_loader, test_loader, validation_loader = create_dataloaders([train_data,test_data,validation_data],batchsize=BATCH_SIZE,shufflelist=[True,True,False])
+    # print("DataLoaders created")
+
+
+    # Set criterion to Cross Entropy and define Adam Optimizer with model parameters and learning rate
+    # criterion_cross_entropy = nn.CrossEntropyLoss()
+    # optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
+    # import time
+
+
+    # Set Scheduler for dynamically Learning Rate adjustment
+    loss_values, eval_values = np.zeros(EPOCH), np.zeros(EPOCH)
+
+    # for epoch in range(EPOCH):
+    #     start = time.time()
+    #     print(f"For {epoch+1} the Scores are: ")
+    #     loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=FREEZE)
+    #     eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
+    #     end = time.time()
+    #     print((end-start),"seconds per epoch needed")
+
+    # plot_metrics_loss_n_acc("x","x","x","x")
+
+    # for epoch in range(EPOCH):
+    #     test_loop(mybert,validation_loader)