Merge branch 'main' of https://gitty.informatik.hs-mannheim.de/3016498/ANLP_WS24_CA2
arman
commit
db3a3474a2
File diff suppressed because one or more lines are too long
220
bert_no_ernie.py
220
bert_no_ernie.py
|
|
@ -61,8 +61,10 @@ class CustomBert(nn.Module):
|
|||
# self.sm = nn.Softmax(dim=1)
|
||||
|
||||
def forward(self, input_ids, attention_mask):
|
||||
seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
|
||||
return self.classifier(self.dropout(seq_out[0]))
|
||||
x = self.bfsc(input_ids, attention_mask = attention_mask)
|
||||
x = self.dropout(x[0])
|
||||
x = self.classifier(x)
|
||||
return x
|
||||
|
||||
|
||||
def freeze_bert_params(self):
|
||||
|
|
@ -73,21 +75,22 @@ class CustomBert(nn.Module):
|
|||
for param in self.bfsc.named_parameters():
|
||||
param[1].requires_grad_(True)
|
||||
|
||||
def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool):
|
||||
def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool=False):
|
||||
model.train()
|
||||
if freeze_bert:
|
||||
model.freeze_bert_params()
|
||||
|
||||
total_loss = 0
|
||||
len_train_loader = len(train_loader)
|
||||
for index,train_batch in enumerate(train_loader):
|
||||
for train_batch in train_loader:
|
||||
|
||||
# Set Gradient to Zero
|
||||
optimizer.zero_grad()
|
||||
|
||||
# Unpack batch values and "push" it to GPU
|
||||
input_ids, att_mask, labels = train_batch.values()
|
||||
# print(f"{input_ids.shape}, {att_mask.shape}, {labels.shape}")
|
||||
# print(f"Iteration {index} of {len_train_loader}")
|
||||
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
|
||||
|
||||
# Feed Model with Data
|
||||
outputs = model(input_ids, attention_mask=att_mask)
|
||||
# print(f"{model.bfsc.}")
|
||||
|
|
@ -96,6 +99,7 @@ def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim
|
|||
loss.backward()
|
||||
optimizer.step()
|
||||
total_loss+=loss.item()
|
||||
|
||||
print(f"Training Loss is {(total_loss/len(train_loader)):.4f}")
|
||||
return (total_loss/len(train_loader))
|
||||
|
||||
|
|
@ -103,108 +107,46 @@ def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:D
|
|||
model.eval()
|
||||
total, correct = 0.0, 0.0
|
||||
total_loss = 0.0
|
||||
best_loss = 10.0
|
||||
best_loss = float("Inf")
|
||||
with torch.no_grad():
|
||||
for val_batch in validation_loader:
|
||||
|
||||
input_ids, att_mask ,labels = val_batch.values()
|
||||
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE)
|
||||
|
||||
outputs = model(input_ids,attention_mask=att_mask)
|
||||
|
||||
loss = criterion(outputs,labels)
|
||||
total_loss += loss.item()
|
||||
|
||||
predictions = torch.argmax(outputs,1)
|
||||
total += labels.size(0)
|
||||
correct += (predictions == labels).sum().item()
|
||||
|
||||
if total_loss/len(validation_loader) < best_loss:
|
||||
best_loss = total_loss/len(validation_loader)
|
||||
torch.save(model,"best_bert_model")
|
||||
print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Test Accuracy {correct/total*100:.4f}%")
|
||||
torch.save(model,"best_bert_model.pt")
|
||||
|
||||
print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Validation Accuracy {correct/total*100:.4f}%")
|
||||
return total_loss/len(validation_loader)
|
||||
|
||||
def test_loop(model:CustomBert, criterion:nn.CrossEntropyLoss, test_loader:DataLoader):
|
||||
def test_loop(model:CustomBert, test_loader:DataLoader):
|
||||
for batch in test_loader:
|
||||
input_ids, att_mask, labels = batch.values()
|
||||
input_ids, att_mask, labels = input_ids.to(DEVICE), att_mask.to(DEVICE), labels.to(DEVICE)
|
||||
with torch.no_grad():
|
||||
model = torch.load("best_bert_model")
|
||||
model.to(DEVICE)
|
||||
output = model(input_ids,att_mask)
|
||||
output.detach().cpu().numpy()
|
||||
labels.detach().cpu().numpy()
|
||||
pred_flat = np.argmax(output,1).flatten()
|
||||
print(accuracy_score(labels,pred_flat))
|
||||
|
||||
def performance_metrics(true_labels,predictions):
|
||||
confusion_matrix(true_labels,predictions)
|
||||
accuracy_score(true_labels,predictions)
|
||||
f1_score(true_labels,predictions)
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# HYPERPARAMETERS
|
||||
# Set Max Epoch Amount
|
||||
EPOCH = 10
|
||||
# DROPOUT-PROBABILITY
|
||||
DROPOUT = 0.1
|
||||
# BATCHSIZE
|
||||
BATCH_SIZE = 16
|
||||
#LEARNING RATE
|
||||
LEARNING_RATE = 1e-5
|
||||
# RANDOM SEED
|
||||
RNDM_SEED = 501
|
||||
|
||||
torch.manual_seed(RNDM_SEED)
|
||||
np.random.seed(RNDM_SEED)
|
||||
torch.cuda.seed_all(RNDM_SEED)
|
||||
|
||||
# Initialize Bert Model with dropout probability and Num End Layers
|
||||
mybert = CustomBert(DROPOUT)
|
||||
print("Bert Initialized")
|
||||
mybert.to(DEVICE)
|
||||
|
||||
|
||||
# Read Raw Data from csv and save as DataFrame
|
||||
df = pd.read_csv("./data/hack.csv",encoding="latin1")
|
||||
print("Raw Data read")
|
||||
|
||||
# Initialize BertTokenizer from Pretrained
|
||||
tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
|
||||
print("Tokenizer Initialized")
|
||||
|
||||
#Split DataFrame into Train and Test Sets
|
||||
train,test = train_test_split(df,random_state=501,test_size=.2)
|
||||
print("Splitted Data in Train and Test Sets")
|
||||
test,val = train_test_split(test,random_state=501,test_size=.5)
|
||||
|
||||
# val = []
|
||||
# Create Custom Datasets for Train and Test
|
||||
train_data = SimpleHumorDataset(tokenizer,train)
|
||||
val_data = SimpleHumorDataset(tokenizer,val)
|
||||
test_data = SimpleHumorDataset(tokenizer,test)
|
||||
print("Custom Datasets created")
|
||||
|
||||
|
||||
# Initialize Dataloader with Train and Test Sets
|
||||
train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
|
||||
validation_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
|
||||
test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
|
||||
print("DataLoaders created")
|
||||
|
||||
# Set criterion to Cross Entropy and define Adam Optimizer with model parameters and learning rate
|
||||
criterion_cross_entropy = nn.CrossEntropyLoss()
|
||||
optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
|
||||
import time
|
||||
# Set Scheduler for dynamically Learning Rate adjustment
|
||||
loss_values = np.zeros(EPOCH)
|
||||
eval_values = np.zeros(EPOCH)
|
||||
freeze = False
|
||||
|
||||
for epoch in range(EPOCH):
|
||||
start = time.time()
|
||||
print(f"For {epoch+1} the Scores are: ")
|
||||
loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=freeze)
|
||||
eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
|
||||
end = time.time()
|
||||
print((end-start),"seconds per epoch needed")
|
||||
def plot_metrics_loss_n_acc(train_loss,validation_loss,train_acc,validation_acc):
|
||||
"""
|
||||
Method that plots Loss and Accuracy of Training and Validation Data used in given modelinstance
|
||||
"""
|
||||
# Visualize Training Loss
|
||||
# plt.plot(loss_values)
|
||||
# plt.plot(eval_values)
|
||||
|
|
@ -214,5 +156,111 @@ if __name__ == "__main__":
|
|||
# plt.xlabel("Num Epochs")
|
||||
# plt.ylabel("Total Loss of Epoch")
|
||||
# plt.show()
|
||||
for epoch in range(EPOCH):
|
||||
test_loop(mybert,criterion_cross_entropy,validation_loader)
|
||||
pass
|
||||
|
||||
def plot_test_metrics(accuracy):
|
||||
"""
|
||||
Plot Test Metrics of Model (Confiuson Matrix, Accuracy)
|
||||
"""
|
||||
plt.plot(accuracy)
|
||||
plt.hlines(np.mean(accuracy),0,len(accuracy),'red','dotted','Mean Accuracy %d'.format(np.mean(accuracy)))
|
||||
plt.title("Accuracy of Test")
|
||||
plt.xlabel("Num Epochs")
|
||||
plt.ylabel("Accurcy 0.0 - 1.0")
|
||||
plt.grid(True)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# def performance_metrics(true_labels,predictions):
|
||||
# confusion_matrix(true_labels,predictions)
|
||||
# accuracy_score(true_labels,predictions)
|
||||
# f1_score(true_labels,predictions)
|
||||
# pass
|
||||
|
||||
def create_datasets(tokenizer:AutoTokenizer,dataframe:pd.DataFrame,train_split_ratio:float,val:bool=False)->tuple[SimpleHumorDataset,SimpleHumorDataset,SimpleHumorDataset]|tuple[SimpleHumorDataset,SimpleHumorDataset]:
|
||||
if train_split_ratio > 1.0:
|
||||
raise AssertionError("Trainsplit sollte kleiner(-gleich) 1.0 sein")
|
||||
train,test = train_test_split(dataframe,train_size=train_split_ratio,random_state=501)
|
||||
if val:
|
||||
test,validation = train_test_split(test,train_size=.5,random_state=501)
|
||||
return SimpleHumorDataset(tokenizer,train), SimpleHumorDataset(tokenizer,test), SimpleHumorDataset(tokenizer,validation)
|
||||
return SimpleHumorDataset(tokenizer,train), SimpleHumorDataset(tokenizer,test)
|
||||
|
||||
def create_dataloaders(datasets:tuple|list,batchsize:int,shufflelist:list):
|
||||
train_loader = DataLoader(datasets[0],batchsize,shuffle=shufflelist[0])
|
||||
test_loader = DataLoader(datasets[1],batchsize,shuffle=shufflelist[1])
|
||||
if len(datasets) == 3:
|
||||
return train_loader, test_loader, DataLoader(datasets[2],batchsize,shuffle=shufflelist[2])
|
||||
return train_loader, test_loader
|
||||
|
||||
|
||||
# if __name__ == "__main__":
|
||||
|
||||
# # HYPERPARAMETERS
|
||||
# # Set Max Epoch Amount
|
||||
# EPOCH = 10
|
||||
# # DROPOUT-PROBABILITY
|
||||
# DROPOUT = 0.1
|
||||
# # BATCHSIZE
|
||||
# BATCH_SIZE = 16
|
||||
# #LEARNING RATE
|
||||
# LEARNING_RATE = 1e-5
|
||||
# # RANDOM SEED
|
||||
# RNDM_SEED = 501
|
||||
# # FREEZE Bert Layers
|
||||
# FREEZE = True
|
||||
|
||||
# torch.manual_seed(RNDM_SEED)
|
||||
# np.random.seed(RNDM_SEED)
|
||||
# torch.cuda.manual_seed_all(RNDM_SEED)
|
||||
|
||||
|
||||
# Initialize Bert Model with dropout probability and port to DEVICE
|
||||
# mybert = CustomBert(DROPOUT)
|
||||
# print("Bert Initialized")
|
||||
# mybert.to(DEVICE)
|
||||
|
||||
|
||||
# Read Raw Data from csv and save as DataFrame
|
||||
# df = pd.read_csv("./data/hack.csv",encoding="latin1")
|
||||
# print("Raw Data read")
|
||||
|
||||
|
||||
# Initialize BertTokenizer from Pretrained
|
||||
# tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
|
||||
# print("Tokenizer Initialized")
|
||||
|
||||
|
||||
# Split DataFrame into Train and Test Sets
|
||||
# Create Custom Datasets for Train and Test
|
||||
# train_data,test_data,validation_data = create_datasets(tokenizer,df,.7,True)
|
||||
# print("Splitted Data in Train and Test Sets")
|
||||
# print("Custom Datasets created")
|
||||
|
||||
|
||||
# Initialize Dataloader with Train and Test Sets
|
||||
# train_loader, test_loader, validation_loader = create_dataloaders([train_data,test_data,validation_data],batchsize=BATCH_SIZE,shufflelist=[True,True,False])
|
||||
# print("DataLoaders created")
|
||||
|
||||
|
||||
# Set criterion to Cross Entropy and define Adam Optimizer with model parameters and learning rate
|
||||
# criterion_cross_entropy = nn.CrossEntropyLoss()
|
||||
# optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
|
||||
# import time
|
||||
|
||||
|
||||
# Set Scheduler for dynamically Learning Rate adjustment
|
||||
loss_values, eval_values = np.zeros(EPOCH), np.zeros(EPOCH)
|
||||
|
||||
# for epoch in range(EPOCH):
|
||||
# start = time.time()
|
||||
# print(f"For {epoch+1} the Scores are: ")
|
||||
# loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=FREEZE)
|
||||
# eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
|
||||
# end = time.time()
|
||||
# print((end-start),"seconds per epoch needed")
|
||||
|
||||
# plot_metrics_loss_n_acc("x","x","x","x")
|
||||
|
||||
# for epoch in range(EPOCH):
|
||||
# test_loop(mybert,validation_loader)
|
||||
Loading…
Reference in New Issue