diff --git a/bert_no_ernie.py b/bert_no_ernie.py index 01e82b8..288dafd 100644 --- a/bert_no_ernie.py +++ b/bert_no_ernie.py @@ -7,7 +7,7 @@ from torch.utils.data import Dataset, DataLoader from sklearn.metrics import accuracy_score, confusion_matrix from sklearn.model_selection import train_test_split # Bert imports -from transformers import BertForSequenceClassification, BertTokenizer, AdamW +from transformers import BertForSequenceClassification, BertTokenizer #Default imports (pandas, numpy, matplotlib, etc.) import pandas as pd import numpy as np @@ -21,7 +21,7 @@ else: class SimpleHumorDataset(Dataset): - def __init__(self,tokenizer,dataframe,max_length=256): + def __init__(self,tokenizer,dataframe,max_length=280): super().__init__() self.tokenizer = tokenizer self.max_length = max_length @@ -54,75 +54,59 @@ class SimpleHumorDataset(Dataset): def __len__(self): return len(self.labels) - class CustomBert(nn.Module): - def __init__(self,dropout): + def __init__(self): super(CustomBert,self).__init__() #Bert + Custom Layers (Not a tuple any longer -- idk why) self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased") - self.dropout = nn.Dropout(dropout) self.classifier = nn.Linear(2,2) - self.sm = nn.Sigmoid() + self.sm = nn.Softmax(dim=1) def forward(self, input_ids, attention_mask): seq_out = self.bfsc(input_ids, attention_mask = attention_mask) - x = self.dropout(seq_out.logits) - x = self.classifier(x) + x = self.classifier(seq_out.logits) return self.sm(x) def training_loop(model,criterion,optimizer,train_loader): - model.to(DEVICE) - model.train() - total_loss = 0 - for index, train_batch in enumerate(train_loader): - # Set Gradient to Zero - optimizer.zero_grad() - # Unpack batch values and "push" it to GPU - input_ids, att_mask, labels,_ = train_batch.values() - input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) - # Feed Model with Data - outputs = model(input_ids, attention_mask=att_mask)#, labels=labels) - print(f"{index}::{len(train_loader)} -- Output Tensor: {outputs.shape}, Labels {labels.shape}") - loss = criterion(outputs,labels) - loss.backward() - optimizer.step() - total_loss+=loss.item() - print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") - return (total_loss/len(train_loader)) + torch.cuda.empty_cache() + model.train() + total_loss = 0 + + for train_batch in train_loader: + # Set Gradient to Zero + optimizer.zero_grad() + # Unpack batch values and "push" it to GPU + input_ids, att_mask, labels,_ = train_batch.values() + input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) + # Feed Model with Data + outputs = model(input_ids, attention_mask=att_mask) + loss = criterion(outputs,labels) + loss.backward() + optimizer.step() + total_loss+=loss.item() + print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") + return (total_loss/len(train_loader)) def eval_loop(model,criterion,validation_loader): - model.eval() - total, correct = 0.0, 0.0 - total_loss = 0.0 - with torch.no_grad(): - for val_batch in validation_loader: - input_ids, att_mask ,labels,_ = val_batch.values() - input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) - outputs = model(input_ids,attention_mask=att_mask) - loss = criterion(outputs,labels) - total_loss += loss.item() - predictions = torch.argmax(outputs,1) - print(outputs.squeeze(0)) - print(f"Prediction: {predictions}. VS Actual {labels}") - print(predictions == labels) - total += labels.size(0) - correct += (predictions == labels).sum().item() - print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%") - return total_loss/len(validation_loader) + torch.cuda.empty_cache() + model.eval() + total, correct = 0.0, 0.0 + total_loss = 0.0 + with torch.no_grad(): + for val_batch in validation_loader: + input_ids, att_mask ,labels,_ = val_batch.values() + input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) + outputs = model(input_ids,attention_mask=att_mask) + loss = criterion(outputs,labels) + total_loss += loss.item() + predictions = torch.argmax(outputs,1) + total += labels.size(0) + correct += (predictions == labels).sum().item() + print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%") + return total_loss/len(validation_loader) -def generate_tokens(tokenizer,raw_data): - return tokenizer.encode_plus( - raw_data, - add_special_tokens=True, - padding="max_length", - return_attention_mask = True, - return_token_type_ids = False, - max_length=128, - truncation = True, - return_tensors = 'pt' - ) if __name__ == "__main__": torch.manual_seed(501) @@ -132,12 +116,13 @@ if __name__ == "__main__": # DROPOUT-PROBABILITY DROPOUT = 0.1 # BATCHSIZE - BATCH_SIZE = 8 + BATCH_SIZE = 32 #LEARNING RATE LEARNING_RATE = 1e-5 # Initialize Bert Model with dropout probability and Num End Layers - mybert = CustomBert(DROPOUT) + mybert = CustomBert() print("Bert Initialized") + mybert.to(DEVICE) # Read Raw Data from csv and save as DataFrame @@ -169,17 +154,16 @@ if __name__ == "__main__": # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate criterion_bce = nn.CrossEntropyLoss() - optimizer_adamW = optim.AdamW(mybert.parameters(), lr = LEARNING_RATE) + optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE) # Set Scheduler for dynamically Learning Rate adjustment # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) - loss_values = [] - eval_values = [] + loss_values = np.zeros(EPOCH) + eval_values = np.zeros(EPOCH) for epoch in range(EPOCH): print(f"For {epoch+1} the Scores are: ") - loss_values.append(training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader)) - eval_values.append(eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader)) - # scheduler.step(min(eval_values)) + loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader) + eval_values[epoch] = eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader) # Visualize Training Loss # plt.plot(loss_values)