3016498
/
ANLP_WS24_CA2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Instand GDDR Filler, use with Caution

main
NIls Rekus 2025-02-12 18:03:21 +01:00
parent f493d9e429
commit ed9be773e1
1 changed files with 59 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
bert_no_ernie.py
View File

@ -6,10 +6,8 @@ from torch.utils.data import Dataset, DataLoader
# scikit-learn Imports
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.model_selection import train_test_split
#Gensim Imports
import gensim
# Bert improts
from transformers import BertForSequenceClassification, BertTokenizer, BertPreTrainedModel, AdamW
# Bert imports
from transformers import BertForSequenceClassification, BertTokenizer, AdamW
#Default imports (pandas, numpy, matplotlib, etc.)
import pandas as pd
import numpy as np
@ -28,15 +26,16 @@ class SimpleHumorDataset(Dataset):
self.tokenizer = tokenizer
self.max_length = max_length
self.text = dataframe['text'].tolist()
self.labels = dataframe['is_humor'].unique().tolist()
self.labels = dataframe['is_humor'].tolist()
def __getitem__(self,idx):
text = self.text
labels = self.labels
text = self.text[idx]
labels = self.labels[idx]
encoding = self.tokenizer.encode_plus(
text,
add_special_tokens=True,
padding="max_length",
trunction = True,
# trunction = True,
return_attention_mask = True,
return_token_type_ids = False,
max_length=self.max_length,
@ -49,7 +48,7 @@ class SimpleHumorDataset(Dataset):
return {
'input_ids': torch.as_tensor(input_ids,dtype=torch.long),
'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long),
'labels':torch.as_tensor(self.labels,dtype=torch.float),
'labels':torch.as_tensor(labels,dtype=torch.long),
'text':text
}
@ -58,24 +57,20 @@ class SimpleHumorDataset(Dataset):
class CustomBert(nn.Module):
def __init__(self,dropout,num_layers=2):
def __init__(self,dropout):
super(CustomBert,self).__init__()
self.bfsc = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"),
self.bert_model = self.bfsc[0]
# Add Custom Layers
self.dropout = nn.Dropout(dropout),
self.dropout = self.dropout[0]
self.ln1 = nn.Linear(2,2),
self.ln1 = self.ln1[0]
self.sm1 = nn.Sigmoid()
#Bert + Custom Layers (Not a tuple any longer -- idk why)
self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
self.dropout = nn.Dropout(dropout)
self.classifier = nn.Linear(2,2)
self.sm = nn.Sigmoid()
def forward(self, input_ids, attention_mask):
seq_out = self.bert_model(input_ids, attention_mask = attention_mask)
seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
x = self.dropout(seq_out.logits)
x = self.ln1(x)
return self.sm1(x)
x = self.classifier(x)
return self.sm(x)
def training_loop(model,criterion,optimizer,train_loader):
model.to(DEVICE)
@ -88,9 +83,9 @@ def training_loop(model,criterion,optimizer,train_loader):
input_ids, att_mask, labels,_ = train_batch.values()
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
# Feed Model with Data
outputs = model(input_ids, attention_mask=att_mask)
# print(f"Output Tensor: {outputs}")
loss = criterion(outputs,labels.float())
outputs = model(input_ids, attention_mask=att_mask)#, labels=labels)
print(f"{index}::{len(train_loader)} -- Output Tensor: {outputs.shape}, Labels {labels.shape}")
loss = criterion(outputs,labels)
loss.backward()
optimizer.step()
total_loss+=loss.item()
@ -99,8 +94,8 @@ def training_loop(model,criterion,optimizer,train_loader):
def eval_loop(model,criterion,validation_loader):
model.eval()
total, correct = 0.0, 0.0
total_loss = 0.0
total_acc = 0.0
with torch.no_grad():
for val_batch in validation_loader:
input_ids, att_mask ,labels,_ = val_batch.values()
@ -108,9 +103,14 @@ def eval_loop(model,criterion,validation_loader):
outputs = model(input_ids,attention_mask=att_mask)
loss = criterion(outputs,labels)
total_loss += loss.item()
predictions = torch.argmax(outputs,dim=1)
total_acc += (predictions == labels).sum().item()
print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {total_acc/len(validation_loader)*100}%")
predictions = torch.argmax(outputs,1)
print(outputs.squeeze(0))
print(f"Prediction: {predictions}. VS Actual {labels}")
print(predictions == labels)
total += labels.size(0)
correct += (predictions == labels).sum().item()
print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%")
return total_loss/len(validation_loader)
def generate_tokens(tokenizer,raw_data):
return tokenizer.encode_plus(
@ -126,12 +126,19 @@ def generate_tokens(tokenizer,raw_data):
if __name__ == "__main__":
torch.manual_seed(501)
# HYPERPARAMETERS
# Set Max Epoch Amount
EPOCH = 5
# DROPOUT-PROBABILITY
DROPOUT = 0.1
# BATCHSIZE
BATCH_SIZE = 8
#LEARNING RATE
LEARNING_RATE = 1e-5
# Initialize Bert Model with dropout probability and Num End Layers
mybert = CustomBert(0.1)
mybert = CustomBert(DROPOUT)
print("Bert Initialized")
# Set Max Epoch Amount
EPOCH = 50
# Read Raw Data from csv and save as DataFrame
df = pd.read_csv("./data/hack.csv",encoding="latin1")
@ -143,40 +150,43 @@ if __name__ == "__main__":
print("Tokenizer Initialized")
#Split DataFrame into Train and Test Sets
train,test = train_test_split(df,random_state=501,test_size=.2)
print("Splitted Data in Train and Test Sets")
# val = []
# Create Custom Datasets for Train and Test
train_data = SimpleHumorDataset(tokenizer,train)
# val_data = SimpleHumorDataset(tokenizer,val)
test_data = SimpleHumorDataset(tokenizer,test)
print("Custom Datasets created")
# Initialize Dataloader with Train and Test Sets
train_loader = DataLoader(dataset=train_data,batch_size=16,shuffle=True)
test_loader = DataLoader(dataset=test_data,batch_size=len(test_data))
train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
# val_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
print("DataLoaders created")
# Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
criterion_bce = nn.BCELoss()
optimizer_adam = optim.Adam(mybert.parameters(), lr = 3e-5)
criterion_bce = nn.CrossEntropyLoss()
optimizer_adamW = optim.AdamW(mybert.parameters(), lr = LEARNING_RATE)
# Set Scheduler for dynamically Learning Rate adjustment
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
loss_values = []
eval_values = []
for epoch in range(EPOCH):
print(f"For {epoch+1} the Scores are: ")
loss_values.append(training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader))
# bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader)
scheduler.step(.1)
loss_values.append(training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader))
eval_values.append(eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader))
# scheduler.step(min(eval_values))
# Visualize Training Loss
plt.plot(loss_values)
# plt.plot(loss_values)
plt.plot(eval_values)
plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
plt.title("Training Loss")
plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")
plt.title("Test Loss")
plt.xlabel("Num Epochs")
plt.ylabel("Total Loss of Epoch")
plt.show()