help me, im under the water

bert_no_ernie.py

@@ -7,7 +7,7 @@ from torch.utils.data import Dataset, DataLoader
 from sklearn.metrics import accuracy_score, confusion_matrix
 from sklearn.model_selection import train_test_split
 # Bert imports
-from transformers import BertForSequenceClassification, BertTokenizer, AdamW
+from transformers import BertForSequenceClassification, BertTokenizer
 #Default imports (pandas, numpy, matplotlib, etc.)
 import pandas as pd
 import numpy as np
@@ -21,7 +21,7 @@ else:
 
 
 class SimpleHumorDataset(Dataset):
-    def __init__(self,tokenizer,dataframe,max_length=256):
+    def __init__(self,tokenizer,dataframe,max_length=280):
         super().__init__()
         self.tokenizer = tokenizer
         self.max_length = max_length
@@ -55,36 +55,33 @@ class SimpleHumorDataset(Dataset):
     def __len__(self):
         return len(self.labels)
 
-
 class CustomBert(nn.Module):
-    def __init__(self,dropout):
+    def __init__(self):
         super(CustomBert,self).__init__()
         
         #Bert + Custom Layers (Not a tuple any longer -- idk why)
         self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
-        self.dropout = nn.Dropout(dropout)
         self.classifier = nn.Linear(2,2)
-        self.sm = nn.Sigmoid()
+        self.sm = nn.Softmax(dim=1)
          
     def forward(self, input_ids, attention_mask):
         seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
-        x = self.dropout(seq_out.logits)
-        x = self.classifier(x)
+        x = self.classifier(seq_out.logits)
         return self.sm(x)
     
 def training_loop(model,criterion,optimizer,train_loader):
-        model.to(DEVICE)
+    torch.cuda.empty_cache()
     model.train()
     total_loss = 0
-        for index, train_batch in enumerate(train_loader):
+
+    for train_batch in train_loader:
         # Set Gradient to Zero
         optimizer.zero_grad()
         # Unpack batch values and "push" it to GPU
         input_ids, att_mask, labels,_ = train_batch.values()
         input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
         # Feed Model with Data
-            outputs = model(input_ids, attention_mask=att_mask)#, labels=labels)
-            print(f"{index}::{len(train_loader)} -- Output Tensor: {outputs.shape}, Labels {labels.shape}")
+        outputs = model(input_ids, attention_mask=att_mask)
         loss = criterion(outputs,labels)
         loss.backward()
         optimizer.step()
@@ -93,6 +90,7 @@ def training_loop(model,criterion,optimizer,train_loader):
     return (total_loss/len(train_loader))       
 
 def eval_loop(model,criterion,validation_loader):
+    torch.cuda.empty_cache()
     model.eval()
     total, correct = 0.0, 0.0
     total_loss = 0.0
@@ -104,25 +102,11 @@ def eval_loop(model,criterion,validation_loader):
             loss = criterion(outputs,labels)
             total_loss += loss.item()
             predictions = torch.argmax(outputs,1)
-                print(outputs.squeeze(0))
-                print(f"Prediction: {predictions}. VS Actual {labels}")
-                print(predictions == labels) 
             total += labels.size(0)
             correct += (predictions == labels).sum().item()
     print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total}%")
     return total_loss/len(validation_loader)
 
-def generate_tokens(tokenizer,raw_data):
-    return tokenizer.encode_plus(
-            raw_data,
-            add_special_tokens=True,
-            padding="max_length",
-            return_attention_mask = True,
-            return_token_type_ids = False,
-            max_length=128,
-            truncation = True,
-            return_tensors = 'pt'
-            )
 
 if __name__ == "__main__":
     torch.manual_seed(501)
@@ -132,12 +116,13 @@ if __name__ == "__main__":
     # DROPOUT-PROBABILITY
     DROPOUT = 0.1
     # BATCHSIZE
-    BATCH_SIZE = 8
+    BATCH_SIZE = 32
     #LEARNING RATE
     LEARNING_RATE = 1e-5
     # Initialize Bert Model with dropout probability and Num End Layers
-    mybert = CustomBert(DROPOUT)
+    mybert = CustomBert()
     print("Bert Initialized")
+    mybert.to(DEVICE)
 
 
     # Read Raw Data from csv and save as DataFrame
@@ -169,17 +154,16 @@ if __name__ == "__main__":
 
     # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
     criterion_bce = nn.CrossEntropyLoss()
-    optimizer_adamW = optim.AdamW(mybert.parameters(), lr = LEARNING_RATE)
+    optimizer_adamW = optim.Adam(mybert.parameters(), lr = LEARNING_RATE)
 
     # Set Scheduler for dynamically Learning Rate adjustment
     # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
-    loss_values = []
-    eval_values = []
+    loss_values = np.zeros(EPOCH)
+    eval_values = np.zeros(EPOCH)
     for epoch in range(EPOCH):
         print(f"For {epoch+1} the Scores are: ")
-        loss_values.append(training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader))
-        eval_values.append(eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader))
-        # scheduler.step(min(eval_values))
+        loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_bce,train_loader=train_loader)
+        eval_values[epoch] = eval_loop(mybert,criterion=criterion_bce,validation_loader=test_loader)
     
     # Visualize Training Loss 
     # plt.plot(loss_values)