diff --git a/bert_no_ernie.py b/bert_no_ernie.py index cc6b8da..a1befca 100644 --- a/bert_no_ernie.py +++ b/bert_no_ernie.py @@ -89,12 +89,13 @@ def training_loop(model,criterion,optimizer,train_loader): input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) # Feed Model with Data outputs = model(input_ids, attention_mask=att_mask) - print(f"Output Tensor: {outputs}") + # print(f"Output Tensor: {outputs}") loss = criterion(outputs,labels.float()) loss.backward() optimizer.step() total_loss+=loss.item() - print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") + print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") + return (total_loss/len(train_loader)) def eval_loop(model,criterion,validation_loader): model.eval() @@ -124,13 +125,13 @@ def generate_tokens(tokenizer,raw_data): ) if __name__ == "__main__": - + torch.manual_seed(501) # Initialize Bert Model with dropout probability and Num End Layers mybert = CustomBert(0.1) print("Bert Initialized") - + # Set Max Epoch Amount - EPOCH = 2 + EPOCH = 50 # Read Raw Data from csv and save as DataFrame df = pd.read_csv("./data/hack.csv",encoding="latin1") @@ -160,14 +161,22 @@ if __name__ == "__main__": print("DataLoaders created") # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate - criterion_bce = nn.CrossEntropyLoss() - optimizer_adam = optim.Adam(mybert.parameters(), lr = 1e-5) + criterion_bce = nn.BCELoss() + optimizer_adam = optim.Adam(mybert.parameters(), lr = 3e-5) # Set Scheduler for dynamically Learning Rate adjustment - # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) - + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) + loss_values = [] for epoch in range(EPOCH): print(f"For {epoch+1} the Scores are: ") - training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader) + loss_values.append(training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader)) # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader) - # scheduler.step() \ No newline at end of file + scheduler.step(.1) + + # Visualize Training Loss + plt.plot(loss_values) + plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss") + plt.title("Training Loss") + plt.xlabel("Num Epochs") + plt.ylabel("Total Loss of Epoch") + plt.show() \ No newline at end of file