3016498
/
ANLP_WS24_CA2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

please try me out senpai

main
NIls Rekus 2025-02-14 17:49:12 +01:00
parent ffc959e884
commit b77bdd21b3
1 changed files with 80 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
bert_no_ernie.py
View File

@ -4,7 +4,7 @@ import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
# scikit-learn Imports
# from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
from sklearn.model_selection import train_test_split
# Bert imports
from transformers import BertForSequenceClassification, AutoTokenizer
@ -25,8 +25,8 @@ class SimpleHumorDataset(Dataset):
super(SimpleHumorDataset,self).__init__()
self.tokenizer = tokenizer
self.max_length = max_length
self.text = dataframe['text'].to_list()
self.labels = dataframe['is_humor'].to_list()
self.text = dataframe['text'].to_numpy()
self.labels = dataframe['is_humor'].to_numpy()
def __getitem__(self,idx:int):
text = self.text[idx]
@ -52,41 +52,58 @@ class SimpleHumorDataset(Dataset):
return len(self.labels)
class CustomBert(nn.Module):
def __init__(self):
def __init__(self,dropout):
super().__init__()
#Bert + Custom Layers (Not a tuple any longer -- idk why)
self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
self.dropout = nn.Dropout(dropout)
self.classifier = nn.Linear(2,2)
self.sm = nn.Softmax(dim=1)
# self.sm = nn.Softmax(dim=1)
def forward(self, input_ids, attention_mask):
seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
x = self.classifier(seq_out.logits)
return self.sm(x)
return self.classifier(self.dropout(seq_out[0]))
def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader):
model.train()
total_loss = 0
def freeze_bert_params(self):
for param in self.bfsc.named_parameters():
param[1].requires_grad_(False)
for train_batch in train_loader:
def unfreeze_bert_params(self):
for param in self.bfsc.named_parameters():
param[1].requires_grad_(True)
def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool):
model.train()
if freeze_bert:
model.freeze_bert_params()
total_loss = 0
len_train_loader = len(train_loader)
for index,train_batch in enumerate(train_loader):
# Set Gradient to Zero
optimizer.zero_grad()
# Unpack batch values and "push" it to GPU
input_ids, att_mask, labels = train_batch.values()
# print(f"{input_ids.shape}, {att_mask.shape}, {labels.shape}")
# print(f"Iteration {index} of {len_train_loader}")
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
# Feed Model with Data
outputs = model(input_ids, attention_mask=att_mask)
# print(f"{model.bfsc.}")
# print(f"{outputs.shape}")
loss = criterion(outputs,labels)
loss.backward()
optimizer.step()
total_loss+=loss.item()
print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")
print(f"Training Loss is {(total_loss/len(train_loader)):.4f}")
return (total_loss/len(train_loader))
def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:DataLoader):
model.eval()
total, correct = 0.0, 0.0
total_loss = 0.0
best_loss = 10.0
with torch.no_grad():
for val_batch in validation_loader:
input_ids, att_mask ,labels = val_batch.values()
@ -97,23 +114,50 @@ def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:D
predictions = torch.argmax(outputs,1)
total += labels.size(0)
correct += (predictions == labels).sum().item()
print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total*100}%")
if total_loss/len(validation_loader) < best_loss:
best_loss = total_loss/len(validation_loader)
torch.save(model,"best_bert_model")
print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Test Accuracy {correct/total*100:.4f}%")
return total_loss/len(validation_loader)
def test_loop(model:CustomBert, criterion:nn.CrossEntropyLoss, test_loader:DataLoader):
for batch in test_loader:
input_ids, att_mask, labels = batch.values()
input_ids, att_mask, labels = input_ids.to(DEVICE), att_mask.to(DEVICE), labels.to(DEVICE)
with torch.no_grad():
output = model(input_ids,att_mask)
output.detach().cpu().numpy()
labels.detach().cpu().numpy()
pred_flat = np.argmax(output,1).flatten()
print(accuracy_score(labels,pred_flat))
def performance_metrics(true_labels,predictions):
confusion_matrix(true_labels,predictions)
accuracy_score(true_labels,predictions)
f1_score(true_labels,predictions)
pass
if __name__ == "__main__":
torch.manual_seed(501)
# HYPERPARAMETERS
# Set Max Epoch Amount
EPOCH = 1
EPOCH = 10
# DROPOUT-PROBABILITY
DROPOUT = 0.1
# BATCHSIZE
BATCH_SIZE = 8
BATCH_SIZE = 16
#LEARNING RATE
LEARNING_RATE = 1e-5
# RANDOM SEED
RNDM_SEED = 501
torch.manual_seed(RNDM_SEED)
np.random.seed(RNDM_SEED)
torch.cuda.seed_all(RNDM_SEED)
# Initialize Bert Model with dropout probability and Num End Layers
mybert = CustomBert()
mybert = CustomBert(DROPOUT)
print("Bert Initialized")
mybert.to(DEVICE)
@ -122,27 +166,26 @@ if __name__ == "__main__":
df = pd.read_csv("./data/hack.csv",encoding="latin1")
print("Raw Data read")
# Initialize BertTokenizer from Pretrained
tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
print("Tokenizer Initialized")
# print(tokenizer(df['text'][0],padding=True,truncation=True,max_length=256))
#Split DataFrame into Train and Test Sets
train,test = train_test_split(df,random_state=501,test_size=.2)
print("Splitted Data in Train and Test Sets")
test,val = train_test_split(test,random_state=501,test_size=.5)
# val = []
# Create Custom Datasets for Train and Test
train_data = SimpleHumorDataset(tokenizer,train)
# val_data = SimpleHumorDataset(tokenizer,val)
val_data = SimpleHumorDataset(tokenizer,val)
test_data = SimpleHumorDataset(tokenizer,test)
print("Custom Datasets created")
# Initialize Dataloader with Train and Test Sets
train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
# val_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
validation_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
print("DataLoaders created")
@ -153,20 +196,23 @@ if __name__ == "__main__":
# Set Scheduler for dynamically Learning Rate adjustment
loss_values = np.zeros(EPOCH)
eval_values = np.zeros(EPOCH)
start = time.time()
freeze = False
for epoch in range(EPOCH):
start = time.time()
print(f"For {epoch+1} the Scores are: ")
loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader)
loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=freeze)
eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
end = time.time()
print((end-start),"seconds per epoch needed")
end = time.time()
print((end-start),"seconds per epoch needed")
# Visualize Training Loss
plt.plot(loss_values)
plt.plot(eval_values)
plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")
plt.title("Test Loss")
plt.xlabel("Num Epochs")
plt.ylabel("Total Loss of Epoch")
plt.show()
# plt.plot(loss_values)
# plt.plot(eval_values)
# plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
# plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")
# plt.title("Test Loss")
# plt.xlabel("Num Epochs")
# plt.ylabel("Total Loss of Epoch")
# plt.show()
for epoch in range(EPOCH):
test_loop(mybert,criterion_cross_entropy,validation_loader)