3016498
/
ANLP_WS24_CA2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Welp... 3Min Per Epoch. GPU goes brrrr brrrr.

main
NIls Rekus 2025-02-13 22:03:49 +01:00
parent fed0b46fd4
commit bf79e30900
1 changed files with 32 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
bert_no_ernie.py
View File

@ -4,10 +4,10 @@ import torch.nn as nn
import torch.optim as optim import torch.optim as optim
from torch.utils.data import Dataset, DataLoader from torch.utils.data import Dataset, DataLoader
# scikit-learn Imports # scikit-learn Imports
from sklearn.metrics import accuracy_score, confusion_matrix # from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
# Bert imports # Bert imports
from transformers import BertForSequenceClassification, BertTokenizer from transformers import BertForSequenceClassification, AutoTokenizer
#Default imports (pandas, numpy, matplotlib, etc.) #Default imports (pandas, numpy, matplotlib, etc.)
import pandas as pd import pandas as pd
import numpy as np import numpy as np
@ -21,23 +21,20 @@ else:
class SimpleHumorDataset(Dataset): class SimpleHumorDataset(Dataset):
def __init__(self,tokenizer,dataframe,max_length=280): def __init__(self,tokenizer:AutoTokenizer,dataframe:pd.DataFrame,max_length:int=128):
super().__init__() super(SimpleHumorDataset,self).__init__()
self.tokenizer = tokenizer self.tokenizer = tokenizer
self.max_length = max_length self.max_length = max_length
self.text = dataframe['text'].tolist() self.text = dataframe['text'].to_list()
self.labels = dataframe['is_humor'].tolist() self.labels = dataframe['is_humor'].to_list()
def __getitem__(self,idx): def __getitem__(self,idx:int):
text = self.text[idx] text = self.text[idx]
labels = self.labels[idx] labels = self.labels[idx]
encoding = self.tokenizer.encode_plus( encoding = self.tokenizer(
text, text,
add_special_tokens=True,
padding="max_length", padding="max_length",
# trunction = True,
return_attention_mask = True, return_attention_mask = True,
return_token_type_ids = False,
max_length=self.max_length, max_length=self.max_length,
truncation = True, truncation = True,
return_tensors = 'pt' return_tensors = 'pt'
@ -48,17 +45,15 @@ class SimpleHumorDataset(Dataset):
return { return {
'input_ids': torch.as_tensor(input_ids,dtype=torch.long), 'input_ids': torch.as_tensor(input_ids,dtype=torch.long),
'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long), 'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long),
'labels':torch.as_tensor(labels,dtype=torch.long), 'labels':torch.tensor(labels,dtype=torch.long)
'text':text }
}
def __len__(self): def __len__(self):
return len(self.labels) return len(self.labels)
class CustomBert(nn.Module): class CustomBert(nn.Module):
def __init__(self): def __init__(self):
super(CustomBert,self).__init__() super().__init__()
#Bert + Custom Layers (Not a tuple any longer -- idk why) #Bert + Custom Layers (Not a tuple any longer -- idk why)
self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased") self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
self.classifier = nn.Linear(2,2) self.classifier = nn.Linear(2,2)
@ -69,8 +64,7 @@ class CustomBert(nn.Module):
x = self.classifier(seq_out.logits) x = self.classifier(seq_out.logits)
return self.sm(x) return self.sm(x)
def training_loop(model,criterion,optimizer,train_loader): def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader):
torch.cuda.empty_cache()
model.train() model.train()
total_loss = 0 total_loss = 0
@ -78,7 +72,7 @@ def training_loop(model,criterion,optimizer,train_loader):
# Set Gradient to Zero # Set Gradient to Zero
optimizer.zero_grad() optimizer.zero_grad()
# Unpack batch values and "push" it to GPU # Unpack batch values and "push" it to GPU
input_ids, att_mask, labels,_ = train_batch.values() input_ids, att_mask, labels = train_batch.values()
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
# Feed Model with Data # Feed Model with Data
outputs = model(input_ids, attention_mask=att_mask) outputs = model(input_ids, attention_mask=att_mask)
@ -89,14 +83,13 @@ def training_loop(model,criterion,optimizer,train_loader):
print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")
return (total_loss/len(train_loader)) return (total_loss/len(train_loader))
def eval_loop(model,criterion,validation_loader): def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:DataLoader):
torch.cuda.empty_cache()
model.eval() model.eval()
total, correct = 0.0, 0.0 total, correct = 0.0, 0.0
total_loss = 0.0 total_loss = 0.0
with torch.no_grad(): with torch.no_grad():
for val_batch in validation_loader: for val_batch in validation_loader:
input_ids, att_mask ,labels,_ = val_batch.values() input_ids, att_mask ,labels = val_batch.values()
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE)
outputs = model(input_ids,attention_mask=att_mask) outputs = model(input_ids,attention_mask=att_mask)
loss = criterion(outputs,labels) loss = criterion(outputs,labels)
@ -104,7 +97,7 @@ def eval_loop(model,criterion,validation_loader):
predictions = torch.argmax(outputs,1) predictions = torch.argmax(outputs,1)
total += labels.size(0) total += labels.size(0)
correct += (predictions == labels).sum().item() correct += (predictions == labels).sum().item()
print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%") print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total*100}%")
return total_loss/len(validation_loader) return total_loss/len(validation_loader)
@ -112,11 +105,11 @@ if __name__ == "__main__":
torch.manual_seed(501) torch.manual_seed(501)
# HYPERPARAMETERS # HYPERPARAMETERS
# Set Max Epoch Amount # Set Max Epoch Amount
EPOCH = 5 EPOCH = 1
# DROPOUT-PROBABILITY # DROPOUT-PROBABILITY
DROPOUT = 0.1 DROPOUT = 0.1
# BATCHSIZE # BATCHSIZE
BATCH_SIZE = 32 BATCH_SIZE = 8
#LEARNING RATE #LEARNING RATE
LEARNING_RATE = 1e-5 LEARNING_RATE = 1e-5
# Initialize Bert Model with dropout probability and Num End Layers # Initialize Bert Model with dropout probability and Num End Layers
@ -131,13 +124,14 @@ if __name__ == "__main__":
# Initialize BertTokenizer from Pretrained # Initialize BertTokenizer from Pretrained
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased",do_lower_case=True) tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
print("Tokenizer Initialized") print("Tokenizer Initialized")
# print(tokenizer(df['text'][0],padding=True,truncation=True,max_length=256))
#Split DataFrame into Train and Test Sets #Split DataFrame into Train and Test Sets
train,test = train_test_split(df,random_state=501,test_size=.2) train,test = train_test_split(df,random_state=501,test_size=.2)
print("Splitted Data in Train and Test Sets") print("Splitted Data in Train and Test Sets")
# val = [] # val = []
# Create Custom Datasets for Train and Test # Create Custom Datasets for Train and Test
train_data = SimpleHumorDataset(tokenizer,train) train_data = SimpleHumorDataset(tokenizer,train)
@ -152,21 +146,23 @@ if __name__ == "__main__":
test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False) test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
print("DataLoaders created") print("DataLoaders created")
# Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate # Set criterion to Cross Entropy and define Adam Optimizer with model parameters and learning rate
criterion_bce = nn.CrossEntropyLoss() criterion_cross_entropy = nn.CrossEntropyLoss()
optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE) optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
import time
# Set Scheduler for dynamically Learning Rate adjustment # Set Scheduler for dynamically Learning Rate adjustment
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
loss_values = np.zeros(EPOCH) loss_values = np.zeros(EPOCH)
eval_values = np.zeros(EPOCH) eval_values = np.zeros(EPOCH)
start = time.time()
for epoch in range(EPOCH): for epoch in range(EPOCH):
print(f"For {epoch+1} the Scores are: ")
loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader)
eval_values[epoch] = eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader)
print(f"For {epoch+1} the Scores are: ")
loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader)
eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
end = time.time()
print((end-start),"seconds per epoch needed")
# Visualize Training Loss # Visualize Training Loss
# plt.plot(loss_values) plt.plot(loss_values)
plt.plot(eval_values) plt.plot(eval_values)
plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss") plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss") plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")