Instand GDDR Filler, use with Caution

bert_no_ernie.py

@@ -6,10 +6,8 @@ from torch.utils.data import Dataset, DataLoader
 # scikit-learn Imports
 from sklearn.metrics import accuracy_score, confusion_matrix
 from sklearn.model_selection import train_test_split
-#Gensim Imports
+# Bert imports
-import gensim
+from transformers import BertForSequenceClassification, BertTokenizer, AdamW
-# Bert improts
-from transformers import BertForSequenceClassification, BertTokenizer, BertPreTrainedModel, AdamW
 #Default imports (pandas, numpy, matplotlib, etc.)
 import pandas as pd
 import numpy as np
@@ -28,15 +26,16 @@ class SimpleHumorDataset(Dataset):
         self.tokenizer = tokenizer
         self.max_length = max_length
         self.text = dataframe['text'].tolist()
-        self.labels = dataframe['is_humor'].unique().tolist()
+        self.labels = dataframe['is_humor'].tolist()
+    
     def __getitem__(self,idx):
-        text = self.text    
+        text = self.text[idx]    
-        labels = self.labels
+        labels = self.labels[idx]
         encoding = self.tokenizer.encode_plus(
             text,
             add_special_tokens=True,
             padding="max_length",
-            trunction = True,
+            # trunction = True,
             return_attention_mask = True,
             return_token_type_ids = False,
             max_length=self.max_length,
@@ -49,7 +48,7 @@ class SimpleHumorDataset(Dataset):
         return {
             'input_ids': torch.as_tensor(input_ids,dtype=torch.long),
             'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long),
-            'labels':torch.as_tensor(self.labels,dtype=torch.float),
+            'labels':torch.as_tensor(labels,dtype=torch.long),
             'text':text
         }
     
@@ -58,24 +57,20 @@ class SimpleHumorDataset(Dataset):
     
 
 class CustomBert(nn.Module):
-    def __init__(self,dropout,num_layers=2):
+    def __init__(self,dropout):
         super(CustomBert,self).__init__()
-
+        
-        self.bfsc = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"),
+        #Bert + Custom Layers (Not a tuple any longer -- idk why)
-        self.bert_model = self.bfsc[0]
+        self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
-
+        self.dropout = nn.Dropout(dropout)
-        # Add Custom Layers
+        self.classifier = nn.Linear(2,2)
-        self.dropout = nn.Dropout(dropout),
+        self.sm = nn.Sigmoid()
-        self.dropout = self.dropout[0]
-        self.ln1 = nn.Linear(2,2),
-        self.ln1 = self.ln1[0]
-        self.sm1 = nn.Sigmoid()
          
     def forward(self, input_ids, attention_mask):
-        seq_out = self.bert_model(input_ids, attention_mask = attention_mask)
+        seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
         x = self.dropout(seq_out.logits)
-        x = self.ln1(x)
+        x = self.classifier(x)
-        return self.sm1(x)
+        return self.sm(x)
     
 def training_loop(model,criterion,optimizer,train_loader):
         model.to(DEVICE)
@@ -88,9 +83,9 @@ def training_loop(model,criterion,optimizer,train_loader):
             input_ids, att_mask, labels,_ = train_batch.values()
             input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
             # Feed Model with Data
-            outputs = model(input_ids, attention_mask=att_mask)
+            outputs = model(input_ids, attention_mask=att_mask)#, labels=labels)
-            # print(f"Output Tensor: {outputs}")
+            print(f"{index}::{len(train_loader)} -- Output Tensor: {outputs.shape}, Labels {labels.shape}")
-            loss = criterion(outputs,labels.float())
+            loss = criterion(outputs,labels)
             loss.backward()
             optimizer.step()
             total_loss+=loss.item()
@@ -99,8 +94,8 @@ def training_loop(model,criterion,optimizer,train_loader):
 
 def eval_loop(model,criterion,validation_loader):
         model.eval()
+        total, correct = 0.0, 0.0
         total_loss = 0.0
-        total_acc = 0.0
         with torch.no_grad():
             for val_batch in validation_loader:
                 input_ids, att_mask ,labels,_ = val_batch.values()
@@ -108,9 +103,14 @@ def eval_loop(model,criterion,validation_loader):
                 outputs = model(input_ids,attention_mask=att_mask)
                 loss = criterion(outputs,labels)
                 total_loss += loss.item()
-                predictions = torch.argmax(outputs,dim=1)
+                predictions = torch.argmax(outputs,1)
-                total_acc += (predictions == labels).sum().item()
+                print(outputs.squeeze(0))
-        print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {total_acc/len(validation_loader)*100}%")
+                print(f"Prediction: {predictions}. VS Actual {labels}")
+                print(predictions == labels) 
+                total += labels.size(0)
+                correct += (predictions == labels).sum().item()
+        print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%")
+        return total_loss/len(validation_loader)
 
 def generate_tokens(tokenizer,raw_data):
     return tokenizer.encode_plus(
@@ -126,13 +126,20 @@ def generate_tokens(tokenizer,raw_data):
 
 if __name__ == "__main__":
     torch.manual_seed(501)
-    # Initialize Bert Model with dropout probability and Num End Layers
+    # HYPERPARAMETERS
-    mybert = CustomBert(0.1)
-    print("Bert Initialized")
-    
     # Set Max Epoch Amount
-    EPOCH = 50
+    EPOCH = 5
-   
+    # DROPOUT-PROBABILITY
+    DROPOUT = 0.1
+    # BATCHSIZE
+    BATCH_SIZE = 8
+    #LEARNING RATE
+    LEARNING_RATE = 1e-5
+    # Initialize Bert Model with dropout probability and Num End Layers
+    mybert = CustomBert(DROPOUT)
+    print("Bert Initialized")
+
+
     # Read Raw Data from csv and save as DataFrame
     df = pd.read_csv("./data/hack.csv",encoding="latin1")
     print("Raw Data read")
@@ -143,40 +150,43 @@ if __name__ == "__main__":
     print("Tokenizer Initialized")
 
 
-
     #Split DataFrame into Train and Test Sets
     train,test = train_test_split(df,random_state=501,test_size=.2)
     print("Splitted Data in Train and Test Sets")
-
+    # val = []
-
     # Create Custom Datasets for Train and Test
     train_data = SimpleHumorDataset(tokenizer,train)
+    # val_data = SimpleHumorDataset(tokenizer,val)
     test_data = SimpleHumorDataset(tokenizer,test)
     print("Custom Datasets created")
 
 
    # Initialize Dataloader with Train and Test Sets
-    train_loader = DataLoader(dataset=train_data,batch_size=16,shuffle=True)
+    train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
-    test_loader = DataLoader(dataset=test_data,batch_size=len(test_data))
+    # val_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
+    test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
     print("DataLoaders created")
 
     # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
-    criterion_bce = nn.BCELoss()
+    criterion_bce = nn.CrossEntropyLoss()
-    optimizer_adam = optim.Adam(mybert.parameters(), lr = 3e-5)
+    optimizer_adamW = optim.AdamW(mybert.parameters(), lr = LEARNING_RATE)
 
     # Set Scheduler for dynamically Learning Rate adjustment
-    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
+    # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
     loss_values = []
+    eval_values = []
     for epoch in range(EPOCH):
         print(f"For {epoch+1} the Scores are: ")
-        loss_values.append(training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader))
+        loss_values.append(training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader))
-        # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader)
+        eval_values.append(eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader))
-        scheduler.step(.1)
+        # scheduler.step(min(eval_values))
     
     # Visualize Training Loss 
-    plt.plot(loss_values)
+    # plt.plot(loss_values)
+    plt.plot(eval_values)
     plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
-    plt.title("Training Loss")
+    plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")
+    plt.title("Test Loss")
     plt.xlabel("Num Epochs")
     plt.ylabel("Total Loss of Epoch")
     plt.show()