3016498
/
ANLP_WS24_CA2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Training Loss looks wild. Please Help

main
NIls Rekus 2025-02-11 21:12:44 +01:00
parent 01b971b610
commit 277fe215a3
1 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
bert_no_ernie.py
View File

@ -89,12 +89,13 @@ def training_loop(model,criterion,optimizer,train_loader):
input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
# Feed Model with Data # Feed Model with Data
outputs = model(input_ids, attention_mask=att_mask) outputs = model(input_ids, attention_mask=att_mask)
print(f"Output Tensor: {outputs}") # print(f"Output Tensor: {outputs}")
loss = criterion(outputs,labels.float()) loss = criterion(outputs,labels.float())
loss.backward() loss.backward()
optimizer.step() optimizer.step()
total_loss+=loss.item() total_loss+=loss.item()
print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")
return (total_loss/len(train_loader))
def eval_loop(model,criterion,validation_loader): def eval_loop(model,criterion,validation_loader):
model.eval() model.eval()
@ -124,13 +125,13 @@ def generate_tokens(tokenizer,raw_data):
) )
if __name__ == "__main__": if __name__ == "__main__":
torch.manual_seed(501)
# Initialize Bert Model with dropout probability and Num End Layers # Initialize Bert Model with dropout probability and Num End Layers
mybert = CustomBert(0.1) mybert = CustomBert(0.1)
print("Bert Initialized") print("Bert Initialized")
# Set Max Epoch Amount # Set Max Epoch Amount
EPOCH = 2 EPOCH = 50
# Read Raw Data from csv and save as DataFrame # Read Raw Data from csv and save as DataFrame
df = pd.read_csv("./data/hack.csv",encoding="latin1") df = pd.read_csv("./data/hack.csv",encoding="latin1")
@ -160,14 +161,22 @@ if __name__ == "__main__":
print("DataLoaders created") print("DataLoaders created")
# Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
criterion_bce = nn.CrossEntropyLoss() criterion_bce = nn.BCELoss()
optimizer_adam = optim.Adam(mybert.parameters(), lr = 1e-5) optimizer_adam = optim.Adam(mybert.parameters(), lr = 3e-5)
# Set Scheduler for dynamically Learning Rate adjustment # Set Scheduler for dynamically Learning Rate adjustment
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
loss_values = []
for epoch in range(EPOCH): for epoch in range(EPOCH):
print(f"For {epoch+1} the Scores are: ") print(f"For {epoch+1} the Scores are: ")
training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader) loss_values.append(training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader))
# bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader) # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader)
# scheduler.step() scheduler.step(.1)
# Visualize Training Loss
plt.plot(loss_values)
plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
plt.title("Training Loss")
plt.xlabel("Num Epochs")
plt.ylabel("Total Loss of Epoch")
plt.show()