diff --git a/bert_no_ernie.py b/bert_no_ernie.py index a1befca..01e82b8 100644 --- a/bert_no_ernie.py +++ b/bert_no_ernie.py @@ -6,10 +6,8 @@ from torch.utils.data import Dataset, DataLoader # scikit-learn Imports from sklearn.metrics import accuracy_score, confusion_matrix from sklearn.model_selection import train_test_split -#Gensim Imports -import gensim -# Bert improts -from transformers import BertForSequenceClassification, BertTokenizer, BertPreTrainedModel, AdamW +# Bert imports +from transformers import BertForSequenceClassification, BertTokenizer, AdamW #Default imports (pandas, numpy, matplotlib, etc.) import pandas as pd import numpy as np @@ -28,15 +26,16 @@ class SimpleHumorDataset(Dataset): self.tokenizer = tokenizer self.max_length = max_length self.text = dataframe['text'].tolist() - self.labels = dataframe['is_humor'].unique().tolist() + self.labels = dataframe['is_humor'].tolist() + def __getitem__(self,idx): - text = self.text - labels = self.labels + text = self.text[idx] + labels = self.labels[idx] encoding = self.tokenizer.encode_plus( text, add_special_tokens=True, padding="max_length", - trunction = True, + # trunction = True, return_attention_mask = True, return_token_type_ids = False, max_length=self.max_length, @@ -49,7 +48,7 @@ class SimpleHumorDataset(Dataset): return { 'input_ids': torch.as_tensor(input_ids,dtype=torch.long), 'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long), - 'labels':torch.as_tensor(self.labels,dtype=torch.float), + 'labels':torch.as_tensor(labels,dtype=torch.long), 'text':text } @@ -58,24 +57,20 @@ class SimpleHumorDataset(Dataset): class CustomBert(nn.Module): - def __init__(self,dropout,num_layers=2): + def __init__(self,dropout): super(CustomBert,self).__init__() - - self.bfsc = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"), - self.bert_model = self.bfsc[0] - - # Add Custom Layers - self.dropout = nn.Dropout(dropout), - self.dropout = self.dropout[0] - self.ln1 = nn.Linear(2,2), - self.ln1 = self.ln1[0] - self.sm1 = nn.Sigmoid() + + #Bert + Custom Layers (Not a tuple any longer -- idk why) + self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased") + self.dropout = nn.Dropout(dropout) + self.classifier = nn.Linear(2,2) + self.sm = nn.Sigmoid() def forward(self, input_ids, attention_mask): - seq_out = self.bert_model(input_ids, attention_mask = attention_mask) + seq_out = self.bfsc(input_ids, attention_mask = attention_mask) x = self.dropout(seq_out.logits) - x = self.ln1(x) - return self.sm1(x) + x = self.classifier(x) + return self.sm(x) def training_loop(model,criterion,optimizer,train_loader): model.to(DEVICE) @@ -88,9 +83,9 @@ def training_loop(model,criterion,optimizer,train_loader): input_ids, att_mask, labels,_ = train_batch.values() input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) # Feed Model with Data - outputs = model(input_ids, attention_mask=att_mask) - # print(f"Output Tensor: {outputs}") - loss = criterion(outputs,labels.float()) + outputs = model(input_ids, attention_mask=att_mask)#, labels=labels) + print(f"{index}::{len(train_loader)} -- Output Tensor: {outputs.shape}, Labels {labels.shape}") + loss = criterion(outputs,labels) loss.backward() optimizer.step() total_loss+=loss.item() @@ -99,8 +94,8 @@ def training_loop(model,criterion,optimizer,train_loader): def eval_loop(model,criterion,validation_loader): model.eval() + total, correct = 0.0, 0.0 total_loss = 0.0 - total_acc = 0.0 with torch.no_grad(): for val_batch in validation_loader: input_ids, att_mask ,labels,_ = val_batch.values() @@ -108,9 +103,14 @@ def eval_loop(model,criterion,validation_loader): outputs = model(input_ids,attention_mask=att_mask) loss = criterion(outputs,labels) total_loss += loss.item() - predictions = torch.argmax(outputs,dim=1) - total_acc += (predictions == labels).sum().item() - print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {total_acc/len(validation_loader)*100}%") + predictions = torch.argmax(outputs,1) + print(outputs.squeeze(0)) + print(f"Prediction: {predictions}. VS Actual {labels}") + print(predictions == labels) + total += labels.size(0) + correct += (predictions == labels).sum().item() + print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%") + return total_loss/len(validation_loader) def generate_tokens(tokenizer,raw_data): return tokenizer.encode_plus( @@ -126,13 +126,20 @@ def generate_tokens(tokenizer,raw_data): if __name__ == "__main__": torch.manual_seed(501) - # Initialize Bert Model with dropout probability and Num End Layers - mybert = CustomBert(0.1) - print("Bert Initialized") - + # HYPERPARAMETERS # Set Max Epoch Amount - EPOCH = 50 - + EPOCH = 5 + # DROPOUT-PROBABILITY + DROPOUT = 0.1 + # BATCHSIZE + BATCH_SIZE = 8 + #LEARNING RATE + LEARNING_RATE = 1e-5 + # Initialize Bert Model with dropout probability and Num End Layers + mybert = CustomBert(DROPOUT) + print("Bert Initialized") + + # Read Raw Data from csv and save as DataFrame df = pd.read_csv("./data/hack.csv",encoding="latin1") print("Raw Data read") @@ -143,40 +150,43 @@ if __name__ == "__main__": print("Tokenizer Initialized") - #Split DataFrame into Train and Test Sets train,test = train_test_split(df,random_state=501,test_size=.2) print("Splitted Data in Train and Test Sets") - - + # val = [] # Create Custom Datasets for Train and Test train_data = SimpleHumorDataset(tokenizer,train) + # val_data = SimpleHumorDataset(tokenizer,val) test_data = SimpleHumorDataset(tokenizer,test) print("Custom Datasets created") # Initialize Dataloader with Train and Test Sets - train_loader = DataLoader(dataset=train_data,batch_size=16,shuffle=True) - test_loader = DataLoader(dataset=test_data,batch_size=len(test_data)) + train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True) + # val_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True) + test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False) print("DataLoaders created") # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate - criterion_bce = nn.BCELoss() - optimizer_adam = optim.Adam(mybert.parameters(), lr = 3e-5) + criterion_bce = nn.CrossEntropyLoss() + optimizer_adamW = optim.AdamW(mybert.parameters(), lr = LEARNING_RATE) # Set Scheduler for dynamically Learning Rate adjustment - scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) + # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) loss_values = [] + eval_values = [] for epoch in range(EPOCH): print(f"For {epoch+1} the Scores are: ") - loss_values.append(training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader)) - # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader) - scheduler.step(.1) + loss_values.append(training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader)) + eval_values.append(eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader)) + # scheduler.step(min(eval_values)) # Visualize Training Loss - plt.plot(loss_values) + # plt.plot(loss_values) + plt.plot(eval_values) plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss") - plt.title("Training Loss") + plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss") + plt.title("Test Loss") plt.xlabel("Num Epochs") plt.ylabel("Total Loss of Epoch") plt.show() \ No newline at end of file