From b77bdd21b353396cf88ac4108975b794f0d9e490 Mon Sep 17 00:00:00 2001 From: Nils <1826514@stud.hs-mannheim.de> Date: Fri, 14 Feb 2025 17:49:12 +0100 Subject: [PATCH] please try me out senpai --- bert_no_ernie.py | 114 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 34 deletions(-) diff --git a/bert_no_ernie.py b/bert_no_ernie.py index bd9797c..7114fe7 100644 --- a/bert_no_ernie.py +++ b/bert_no_ernie.py @@ -4,7 +4,7 @@ import torch.nn as nn import torch.optim as optim from torch.utils.data import Dataset, DataLoader # scikit-learn Imports -# from sklearn.metrics import accuracy_score, confusion_matrix +from sklearn.metrics import accuracy_score, confusion_matrix, f1_score from sklearn.model_selection import train_test_split # Bert imports from transformers import BertForSequenceClassification, AutoTokenizer @@ -25,8 +25,8 @@ class SimpleHumorDataset(Dataset): super(SimpleHumorDataset,self).__init__() self.tokenizer = tokenizer self.max_length = max_length - self.text = dataframe['text'].to_list() - self.labels = dataframe['is_humor'].to_list() + self.text = dataframe['text'].to_numpy() + self.labels = dataframe['is_humor'].to_numpy() def __getitem__(self,idx:int): text = self.text[idx] @@ -52,41 +52,58 @@ class SimpleHumorDataset(Dataset): return len(self.labels) class CustomBert(nn.Module): - def __init__(self): + def __init__(self,dropout): super().__init__() #Bert + Custom Layers (Not a tuple any longer -- idk why) self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased") + self.dropout = nn.Dropout(dropout) self.classifier = nn.Linear(2,2) - self.sm = nn.Softmax(dim=1) + # self.sm = nn.Softmax(dim=1) def forward(self, input_ids, attention_mask): seq_out = self.bfsc(input_ids, attention_mask = attention_mask) - x = self.classifier(seq_out.logits) - return self.sm(x) + return self.classifier(self.dropout(seq_out[0])) + -def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader): - model.train() - total_loss = 0 + def freeze_bert_params(self): + for param in self.bfsc.named_parameters(): + param[1].requires_grad_(False) - for train_batch in train_loader: + def unfreeze_bert_params(self): + for param in self.bfsc.named_parameters(): + param[1].requires_grad_(True) + +def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool): + model.train() + if freeze_bert: + model.freeze_bert_params() + + total_loss = 0 + len_train_loader = len(train_loader) + for index,train_batch in enumerate(train_loader): # Set Gradient to Zero optimizer.zero_grad() # Unpack batch values and "push" it to GPU input_ids, att_mask, labels = train_batch.values() + # print(f"{input_ids.shape}, {att_mask.shape}, {labels.shape}") + # print(f"Iteration {index} of {len_train_loader}") input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) # Feed Model with Data outputs = model(input_ids, attention_mask=att_mask) + # print(f"{model.bfsc.}") + # print(f"{outputs.shape}") loss = criterion(outputs,labels) loss.backward() optimizer.step() total_loss+=loss.item() - print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") + print(f"Training Loss is {(total_loss/len(train_loader)):.4f}") return (total_loss/len(train_loader)) def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:DataLoader): model.eval() total, correct = 0.0, 0.0 total_loss = 0.0 + best_loss = 10.0 with torch.no_grad(): for val_batch in validation_loader: input_ids, att_mask ,labels = val_batch.values() @@ -97,23 +114,50 @@ def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:D predictions = torch.argmax(outputs,1) total += labels.size(0) correct += (predictions == labels).sum().item() - print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total*100}%") + if total_loss/len(validation_loader) < best_loss: + best_loss = total_loss/len(validation_loader) + torch.save(model,"best_bert_model") + print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Test Accuracy {correct/total*100:.4f}%") return total_loss/len(validation_loader) +def test_loop(model:CustomBert, criterion:nn.CrossEntropyLoss, test_loader:DataLoader): + for batch in test_loader: + input_ids, att_mask, labels = batch.values() + input_ids, att_mask, labels = input_ids.to(DEVICE), att_mask.to(DEVICE), labels.to(DEVICE) + with torch.no_grad(): + output = model(input_ids,att_mask) + output.detach().cpu().numpy() + labels.detach().cpu().numpy() + pred_flat = np.argmax(output,1).flatten() + print(accuracy_score(labels,pred_flat)) + +def performance_metrics(true_labels,predictions): + confusion_matrix(true_labels,predictions) + accuracy_score(true_labels,predictions) + f1_score(true_labels,predictions) + pass + if __name__ == "__main__": - torch.manual_seed(501) + # HYPERPARAMETERS # Set Max Epoch Amount - EPOCH = 1 + EPOCH = 10 # DROPOUT-PROBABILITY DROPOUT = 0.1 # BATCHSIZE - BATCH_SIZE = 8 + BATCH_SIZE = 16 #LEARNING RATE LEARNING_RATE = 1e-5 + # RANDOM SEED + RNDM_SEED = 501 + + torch.manual_seed(RNDM_SEED) + np.random.seed(RNDM_SEED) + torch.cuda.seed_all(RNDM_SEED) + # Initialize Bert Model with dropout probability and Num End Layers - mybert = CustomBert() + mybert = CustomBert(DROPOUT) print("Bert Initialized") mybert.to(DEVICE) @@ -122,27 +166,26 @@ if __name__ == "__main__": df = pd.read_csv("./data/hack.csv",encoding="latin1") print("Raw Data read") - # Initialize BertTokenizer from Pretrained tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True) print("Tokenizer Initialized") - # print(tokenizer(df['text'][0],padding=True,truncation=True,max_length=256)) #Split DataFrame into Train and Test Sets train,test = train_test_split(df,random_state=501,test_size=.2) print("Splitted Data in Train and Test Sets") + test,val = train_test_split(test,random_state=501,test_size=.5) # val = [] # Create Custom Datasets for Train and Test train_data = SimpleHumorDataset(tokenizer,train) - # val_data = SimpleHumorDataset(tokenizer,val) + val_data = SimpleHumorDataset(tokenizer,val) test_data = SimpleHumorDataset(tokenizer,test) print("Custom Datasets created") # Initialize Dataloader with Train and Test Sets train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True) - # val_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True) + validation_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True) test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False) print("DataLoaders created") @@ -153,20 +196,23 @@ if __name__ == "__main__": # Set Scheduler for dynamically Learning Rate adjustment loss_values = np.zeros(EPOCH) eval_values = np.zeros(EPOCH) - start = time.time() + freeze = False + for epoch in range(EPOCH): - + start = time.time() print(f"For {epoch+1} the Scores are: ") - loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader) + loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=freeze) eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader) - end = time.time() - print((end-start),"seconds per epoch needed") + end = time.time() + print((end-start),"seconds per epoch needed") # Visualize Training Loss - plt.plot(loss_values) - plt.plot(eval_values) - plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss") - plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss") - plt.title("Test Loss") - plt.xlabel("Num Epochs") - plt.ylabel("Total Loss of Epoch") - plt.show() \ No newline at end of file + # plt.plot(loss_values) + # plt.plot(eval_values) + # plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss") + # plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss") + # plt.title("Test Loss") + # plt.xlabel("Num Epochs") + # plt.ylabel("Total Loss of Epoch") + # plt.show() + for epoch in range(EPOCH): + test_loop(mybert,criterion_cross_entropy,validation_loader) \ No newline at end of file