progress

bert_no_ernie.py

@@ -28,7 +28,7 @@ class SimpleHumorDataset(Dataset):
         self.tokenizer = tokenizer
         self.max_length = max_length
         self.text = dataframe['text'].tolist()
-        self.labels = dataframe['is_humor'].tolist()
+        self.labels = dataframe['is_humor'].unique().tolist()
     def __getitem__(self,idx):
         text = self.text    
         labels = self.labels
@@ -49,7 +49,7 @@ class SimpleHumorDataset(Dataset):
         return {
             'input_ids': torch.as_tensor(input_ids,dtype=torch.long),
             'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long),
-            'labels':torch.as_tensor(self.labels,dtype=torch.long),
+            'labels':torch.as_tensor(self.labels,dtype=torch.float),
             'text':text
         }
     
@@ -57,40 +57,54 @@ class SimpleHumorDataset(Dataset):
         return len(self.labels)
     
 
-class BERT(nn.Module):
+class CustomBert(nn.Module):
     def __init__(self,dropout,num_layers=2):
-        super().__init__()
+        super(CustomBert,self).__init__()
-        self.model = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"),
+
-        self.ln1 = nn.Linear(768,num_layers),
+        self.bfsc = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"),
+        self.bert_model = self.bfsc[0]
+
+        # Add Custom Layers
         self.dropout = nn.Dropout(dropout),
+        self.dropout = self.dropout[0]
+        self.ln1 = nn.Linear(2,2),
+        self.ln1 = self.ln1[0]
         self.sm1 = nn.Sigmoid()
          
-    def forward(self,input_ids,attention_mask):
+    def forward(self, input_ids, attention_mask):
-        return self.sm1(self.dropout(self.ln1(self.model[0](input_ids,attention_mask))))
+        seq_out = self.bert_model(input_ids, attention_mask = attention_mask)
-        # return self.model(input_ids)
+        x = self.dropout(seq_out.logits)
+        x = self.ln1(x)
+        return self.sm1(x)
     
-    def train(self,criterion,optimizer,train_loader):
+def training_loop(model,criterion,optimizer,train_loader):
-        self.model[0].train()
+        model.to(DEVICE)
+        model.train()
         total_loss = 0
-        for train_batch in train_loader:
+        for index, train_batch in enumerate(train_loader):
+            # Set Gradient to Zero
             optimizer.zero_grad()
-            input_ids, att_mask, labels,_ = train_batch
+            # Unpack batch values and "push" it to GPU
-            outputs = self.forward(input_ids,att_mask)
+            input_ids, att_mask, labels,_ = train_batch.values()
-            loss = criterion(outputs,labels)
+            input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
+            # Feed Model with Data
+            outputs = model(input_ids, attention_mask=att_mask)
+            print(f"Output Tensor: {outputs}")
+            loss = criterion(outputs,labels.float())
             loss.backward()
             optimizer.step()
             total_loss+=loss.item()
         print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")            
 
-    def eval(model,criterion,validation_loader):
+def eval_loop(model,criterion,validation_loader):
-        model.model[0].eval()
+        model.eval()
         total_loss = 0.0
         total_acc = 0.0
         with torch.no_grad():
             for val_batch in validation_loader:
-                input_ids, att_mask ,labels,_ = val_batch
+                input_ids, att_mask ,labels,_ = val_batch.values()
-                input_ids, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE)
+                input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE)
-                outputs = model(input_ids,att_mask)
+                outputs = model(input_ids,attention_mask=att_mask)
                 loss = criterion(outputs,labels)
                 total_loss += loss.item()
                 predictions = torch.argmax(outputs,dim=1)
@@ -112,33 +126,48 @@ def generate_tokens(tokenizer,raw_data):
 if __name__ == "__main__":
     
     # Initialize Bert Model with dropout probability and Num End Layers
-    bert = BERT(0.1)
+    mybert = CustomBert(0.1)
+    print("Bert Initialized")
 
     # Set Max Epoch Amount
-    EPOCH = 50
+    EPOCH = 2
    
     # Read Raw Data from csv and save as DataFrame
     df = pd.read_csv("./data/hack.csv",encoding="latin1")
+    print("Raw Data read")
+
 
     # Initialize BertTokenizer from Pretrained 
     tokenizer = BertTokenizer.from_pretrained("bert-base-uncased",do_lower_case=True)
+    print("Tokenizer Initialized")
+
+
 
     #Split DataFrame into Train and Test Sets
     train,test = train_test_split(df,random_state=501,test_size=.2)
+    print("Splitted Data in Train and Test Sets")
+
 
     # Create Custom Datasets for Train and Test
     train_data = SimpleHumorDataset(tokenizer,train)
     test_data = SimpleHumorDataset(tokenizer,test)
+    print("Custom Datasets created")
+
 
    # Initialize Dataloader with Train and Test Sets
     train_loader = DataLoader(dataset=train_data,batch_size=16,shuffle=True)
     test_loader = DataLoader(dataset=test_data,batch_size=len(test_data))
+    print("DataLoaders created")
 
     # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate
-    criterion_bce = nn.BCELoss()
+    criterion_bce = nn.CrossEntropyLoss()
-    optimizer_adam = optim.Adam(bert.model[0].parameters(), lr = 1e-5)
+    optimizer_adam = optim.Adam(mybert.parameters(), lr = 1e-5)
+
+    # Set Scheduler for dynamically Learning Rate adjustment
+    # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam)
     
     for epoch in range(EPOCH):
-        print(f"For {epoch} the Scores are: ")
+        print(f"For {epoch+1} the Scores are: ")
-        bert.train(optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader)
+        training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader)
-        bert.eval(criterion=criterion_bce,validation_loader=test_loader)
+        # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader)
+        # scheduler.step()