3016498
/
ANLP_WS24_CA2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

help me, im under the water

main
NIls Rekus 2025-02-12 19:22:20 +01:00
parent ed9be773e1
commit 19ab4fa45f
1 changed files with 47 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
bert_no_ernie.py
View File

@ -7,7 +7,7 @@ from torch.utils.data import Dataset, DataLoader
from sklearn.metrics import accuracy_score, confusion_matrix from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
# Bert imports # Bert imports
from transformers import BertForSequenceClassification, BertTokenizer, AdamW from transformers import BertForSequenceClassification, BertTokenizer
#Default imports (pandas, numpy, matplotlib, etc.) #Default imports (pandas, numpy, matplotlib, etc.)
import pandas as pd import pandas as pd
import numpy as np import numpy as np
@ -21,7 +21,7 @@ else:
class SimpleHumorDataset(Dataset): class SimpleHumorDataset(Dataset):
def __init__(self,tokenizer,dataframe,max_length=256): def __init__(self,tokenizer,dataframe,max_length=280):
super().__init__() super().__init__()
self.tokenizer = tokenizer self.tokenizer = tokenizer
self.max_length = max_length self.max_length = max_length
@ -54,75 +54,59 @@ class SimpleHumorDataset(Dataset):
def __len__(self): def __len__(self):
return len(self.labels) return len(self.labels)
class CustomBert(nn.Module): class CustomBert(nn.Module):
def __init__(self,dropout): def __init__(self):
super(CustomBert,self).__init__() super(CustomBert,self).__init__()
#Bert + Custom Layers (Not a tuple any longer -- idk why) #Bert + Custom Layers (Not a tuple any longer -- idk why)
self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased") self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
self.dropout = nn.Dropout(dropout)
self.classifier = nn.Linear(2,2) self.classifier = nn.Linear(2,2)
self.sm = nn.Sigmoid() self.sm = nn.Softmax(dim=1)
def forward(self, input_ids, attention_mask): def forward(self, input_ids, attention_mask):
seq_out = self.bfsc(input_ids, attention_mask = attention_mask) seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
x = self.dropout(seq_out.logits) x = self.classifier(seq_out.logits)
x = self.classifier(x)
return self.sm(x) return self.sm(x)
def training_loop(model,criterion,optimizer,train_loader): def training_loop(model,criterion,optimizer,train_loader):
model.to(DEVICE) torch.cuda.empty_cache()
model.train() model.train()
total_loss = 0 total_loss = 0
for index, train_batch in enumerate(train_loader):
# Set Gradient to Zero for train_batch in train_loader:
optimizer.zero_grad() # Set Gradient to Zero
# Unpack batch values and "push" it to GPU optimizer.zero_grad()
input_ids, att_mask, labels,_ = train_batch.values() # Unpack batch values and "push" it to GPU
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) input_ids, att_mask, labels,_ = train_batch.values()
# Feed Model with Data input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
outputs = model(input_ids, attention_mask=att_mask)#, labels=labels) # Feed Model with Data
print(f"{index}::{len(train_loader)} -- Output Tensor: {outputs.shape}, Labels {labels.shape}") outputs = model(input_ids, attention_mask=att_mask)
loss = criterion(outputs,labels) loss = criterion(outputs,labels)
loss.backward() loss.backward()
optimizer.step() optimizer.step()
total_loss+=loss.item() total_loss+=loss.item()
print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")
return (total_loss/len(train_loader)) return (total_loss/len(train_loader))
def eval_loop(model,criterion,validation_loader): def eval_loop(model,criterion,validation_loader):
model.eval() torch.cuda.empty_cache()
total, correct = 0.0, 0.0 model.eval()
total_loss = 0.0 total, correct = 0.0, 0.0
with torch.no_grad(): total_loss = 0.0
for val_batch in validation_loader: with torch.no_grad():
input_ids, att_mask ,labels,_ = val_batch.values() for val_batch in validation_loader:
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) input_ids, att_mask ,labels,_ = val_batch.values()
outputs = model(input_ids,attention_mask=att_mask) input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE)
loss = criterion(outputs,labels) outputs = model(input_ids,attention_mask=att_mask)
total_loss += loss.item() loss = criterion(outputs,labels)
predictions = torch.argmax(outputs,1) total_loss += loss.item()
print(outputs.squeeze(0)) predictions = torch.argmax(outputs,1)
print(f"Prediction: {predictions}. VS Actual {labels}") total += labels.size(0)
print(predictions == labels) correct += (predictions == labels).sum().item()
total += labels.size(0) print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%")
correct += (predictions == labels).sum().item() return total_loss/len(validation_loader)
print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%")
return total_loss/len(validation_loader)
def generate_tokens(tokenizer,raw_data):
return tokenizer.encode_plus(
raw_data,
add_special_tokens=True,
padding="max_length",
return_attention_mask = True,
return_token_type_ids = False,
max_length=128,
truncation = True,
return_tensors = 'pt'
)
if __name__ == "__main__": if __name__ == "__main__":
torch.manual_seed(501) torch.manual_seed(501)
@ -132,12 +116,13 @@ if __name__ == "__main__":
# DROPOUT-PROBABILITY # DROPOUT-PROBABILITY
DROPOUT = 0.1 DROPOUT = 0.1
# BATCHSIZE # BATCHSIZE
BATCH_SIZE = 8 BATCH_SIZE = 32
#LEARNING RATE #LEARNING RATE
LEARNING_RATE = 1e-5 LEARNING_RATE = 1e-5
# Initialize Bert Model with dropout probability and Num End Layers # Initialize Bert Model with dropout probability and Num End Layers
mybert = CustomBert(DROPOUT) mybert = CustomBert()
print("Bert Initialized") print("Bert Initialized")
mybert.to(DEVICE)
# Read Raw Data from csv and save as DataFrame # Read Raw Data from csv and save as DataFrame
@ -169,17 +154,16 @@ if __name__ == "__main__":
# Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
criterion_bce = nn.CrossEntropyLoss() criterion_bce = nn.CrossEntropyLoss()
optimizer_adamW = optim.AdamW(mybert.parameters(), lr = LEARNING_RATE) optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
# Set Scheduler for dynamically Learning Rate adjustment # Set Scheduler for dynamically Learning Rate adjustment
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
loss_values = [] loss_values = np.zeros(EPOCH)
eval_values = [] eval_values = np.zeros(EPOCH)
for epoch in range(EPOCH): for epoch in range(EPOCH):
print(f"For {epoch+1} the Scores are: ") print(f"For {epoch+1} the Scores are: ")
loss_values.append(training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader)) loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader)
eval_values.append(eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader)) eval_values[epoch] = eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader)
# scheduler.step(min(eval_values))
# Visualize Training Loss # Visualize Training Loss
# plt.plot(loss_values) # plt.plot(loss_values)