From bf79e30900353893ca400cc76c94588b9fc2df15 Mon Sep 17 00:00:00 2001 From: Nils <1826514@stud.hs-mannheim.de> Date: Thu, 13 Feb 2025 22:03:49 +0100 Subject: [PATCH] Welp... 3Min Per Epoch. GPU goes brrrr brrrr. --- bert_no_ernie.py | 68 +++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/bert_no_ernie.py b/bert_no_ernie.py index 288dafd..bd9797c 100644 --- a/bert_no_ernie.py +++ b/bert_no_ernie.py @@ -4,10 +4,10 @@ import torch.nn as nn import torch.optim as optim from torch.utils.data import Dataset, DataLoader # scikit-learn Imports -from sklearn.metrics import accuracy_score, confusion_matrix +# from sklearn.metrics import accuracy_score, confusion_matrix from sklearn.model_selection import train_test_split # Bert imports -from transformers import BertForSequenceClassification, BertTokenizer +from transformers import BertForSequenceClassification, AutoTokenizer #Default imports (pandas, numpy, matplotlib, etc.) import pandas as pd import numpy as np @@ -21,23 +21,20 @@ else: class SimpleHumorDataset(Dataset): - def __init__(self,tokenizer,dataframe,max_length=280): - super().__init__() + def __init__(self,tokenizer:AutoTokenizer,dataframe:pd.DataFrame,max_length:int=128): + super(SimpleHumorDataset,self).__init__() self.tokenizer = tokenizer self.max_length = max_length - self.text = dataframe['text'].tolist() - self.labels = dataframe['is_humor'].tolist() + self.text = dataframe['text'].to_list() + self.labels = dataframe['is_humor'].to_list() - def __getitem__(self,idx): + def __getitem__(self,idx:int): text = self.text[idx] labels = self.labels[idx] - encoding = self.tokenizer.encode_plus( + encoding = self.tokenizer( text, - add_special_tokens=True, padding="max_length", - # trunction = True, return_attention_mask = True, - return_token_type_ids = False, max_length=self.max_length, truncation = True, return_tensors = 'pt' @@ -48,17 +45,15 @@ class SimpleHumorDataset(Dataset): return { 'input_ids': torch.as_tensor(input_ids,dtype=torch.long), 'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long), - 'labels':torch.as_tensor(labels,dtype=torch.long), - 'text':text - } + 'labels':torch.tensor(labels,dtype=torch.long) + } def __len__(self): return len(self.labels) class CustomBert(nn.Module): def __init__(self): - super(CustomBert,self).__init__() - + super().__init__() #Bert + Custom Layers (Not a tuple any longer -- idk why) self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased") self.classifier = nn.Linear(2,2) @@ -69,8 +64,7 @@ class CustomBert(nn.Module): x = self.classifier(seq_out.logits) return self.sm(x) -def training_loop(model,criterion,optimizer,train_loader): - torch.cuda.empty_cache() +def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader): model.train() total_loss = 0 @@ -78,7 +72,7 @@ def training_loop(model,criterion,optimizer,train_loader): # Set Gradient to Zero optimizer.zero_grad() # Unpack batch values and "push" it to GPU - input_ids, att_mask, labels,_ = train_batch.values() + input_ids, att_mask, labels = train_batch.values() input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) # Feed Model with Data outputs = model(input_ids, attention_mask=att_mask) @@ -89,14 +83,13 @@ def training_loop(model,criterion,optimizer,train_loader): print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") return (total_loss/len(train_loader)) -def eval_loop(model,criterion,validation_loader): - torch.cuda.empty_cache() +def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:DataLoader): model.eval() total, correct = 0.0, 0.0 total_loss = 0.0 with torch.no_grad(): for val_batch in validation_loader: - input_ids, att_mask ,labels,_ = val_batch.values() + input_ids, att_mask ,labels = val_batch.values() input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) outputs = model(input_ids,attention_mask=att_mask) loss = criterion(outputs,labels) @@ -104,7 +97,7 @@ def eval_loop(model,criterion,validation_loader): predictions = torch.argmax(outputs,1) total += labels.size(0) correct += (predictions == labels).sum().item() - print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%") + print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total*100}%") return total_loss/len(validation_loader) @@ -112,11 +105,11 @@ if __name__ == "__main__": torch.manual_seed(501) # HYPERPARAMETERS # Set Max Epoch Amount - EPOCH = 5 + EPOCH = 1 # DROPOUT-PROBABILITY DROPOUT = 0.1 # BATCHSIZE - BATCH_SIZE = 32 + BATCH_SIZE = 8 #LEARNING RATE LEARNING_RATE = 1e-5 # Initialize Bert Model with dropout probability and Num End Layers @@ -131,19 +124,20 @@ if __name__ == "__main__": # Initialize BertTokenizer from Pretrained - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased",do_lower_case=True) + tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True) print("Tokenizer Initialized") - + # print(tokenizer(df['text'][0],padding=True,truncation=True,max_length=256)) #Split DataFrame into Train and Test Sets train,test = train_test_split(df,random_state=501,test_size=.2) print("Splitted Data in Train and Test Sets") + # val = [] # Create Custom Datasets for Train and Test train_data = SimpleHumorDataset(tokenizer,train) # val_data = SimpleHumorDataset(tokenizer,val) test_data = SimpleHumorDataset(tokenizer,test) - print("Custom Datasets created") + print("Custom Datasets created") # Initialize Dataloader with Train and Test Sets @@ -152,21 +146,23 @@ if __name__ == "__main__": test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False) print("DataLoaders created") - # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate - criterion_bce = nn.CrossEntropyLoss() + # Set criterion to Cross Entropy and define Adam Optimizer with model parameters and learning rate + criterion_cross_entropy = nn.CrossEntropyLoss() optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE) - + import time # Set Scheduler for dynamically Learning Rate adjustment - # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) loss_values = np.zeros(EPOCH) eval_values = np.zeros(EPOCH) + start = time.time() for epoch in range(EPOCH): + print(f"For {epoch+1} the Scores are: ") - loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader) - eval_values[epoch] = eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader) - + loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader) + eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader) + end = time.time() + print((end-start),"seconds per epoch needed") # Visualize Training Loss - # plt.plot(loss_values) + plt.plot(loss_values) plt.plot(eval_values) plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss") plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")