Merge branch 'main' of https://gitty.informatik.hs-mannheim.de/3016498/ANLP_WS24_CA2

HumorDataset.py

@@ -5,6 +5,40 @@ import torch
 import numpy as np
 from nltk.tokenize import word_tokenize
 
+class TextRegDataset(torch.utils.data.Dataset):
+    def __init__(self, texts, labels, word_index, max_len=50):
+
+        self.original_indices = labels.index.to_list()
+
+        self.texts = texts.reset_index(drop=True)
+        self.labels = labels.reset_index(drop=True)
+        self.word_index = word_index
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        texts = self.texts[idx]
+        tokens = word_tokenize(texts.lower())
+
+        label = self.labels[idx]
+        
+        # Tokenize and convert to indices
+        input_ids = [self.word_index.get(word, self.word_index['<UNK>']) for word in tokens]
+
+        # Pad or truncate to max_len
+        if len(input_ids) < self.max_len:
+            input_ids += [self.word_index['<PAD>']] * (self.max_len - len(input_ids))
+        else:
+            input_ids = input_ids[:self.max_len]
+        
+        # Convert to PyTorch tensors
+        input_ids = torch.tensor(input_ids, dtype=torch.long)
+        label = torch.tensor(label, dtype=torch.float)
+        
+        return input_ids, label
+    
 class TextDataset(torch.utils.data.Dataset):
     def __init__(self, texts, labels, word_index, max_len=50):
 

TEST_CNN_2.py

@@ -5,7 +5,7 @@ from torch.utils.data import DataLoader
 from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
 from tqdm import tqdm  # Fortschrittsbalken-Bibliothek
 from dataset_generator import create_embedding_matrix, split_data
-from HumorDataset import TextDataset
+from HumorDataset import TextRegDataset
 import numpy as np
 import pandas as pd
 import os
@@ -20,7 +20,7 @@ params = {
     "learning_rate": 0.001,
     "epochs": 25,
     "glove_path": 'data/glove.6B.100d.txt',  # Pfad zu GloVe
-    "max_len": 50,
+    "max_len": 280,
     "test_size": 0.1,
     "val_size": 0.1,
     "patience": 5,
@@ -171,9 +171,9 @@ visualize_data_distribution(y)
 data_split = split_data(X, y, test_size=params["test_size"], val_size=params["val_size"])
 
 # Dataset und DataLoader
-train_dataset = TextDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=params["max_len"])
+train_dataset = TextRegDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=params["max_len"])
-val_dataset = TextDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=params["max_len"])
+val_dataset = TextRegDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=params["max_len"])
-test_dataset = TextDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=params["max_len"])
+test_dataset = TextRegDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=params["max_len"])
 
 train_loader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True)
 val_loader = DataLoader(val_dataset, batch_size=params["batch_size"], shuffle=False)
@@ -187,7 +187,10 @@ model = EnhancedCNNRegressor(
     num_filters=params["num_filters"],
     embedding_matrix=embedding_matrix,
     dropout=params["dropout"]
-).to(device)
+)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = model.to(device)
 
 criterion = nn.MSELoss()
 optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"])
@@ -340,3 +343,15 @@ test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds))
 test_mae = mean_absolute_error(test_labels, test_preds)
 test_r2 = r2_score(test_labels, test_preds)
 print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}")
+
+
+# plot distribution of predicted values and true values
+plt.figure(figsize=(10, 6))
+plt.hist(test_labels, bins=20, color='skyblue', edgecolor='black', alpha=0.7, label='True Values')
+plt.hist(test_preds, bins=20, color='salmon', edgecolor='black', alpha=0.7, label='Predicted Values')
+plt.title('Distribution of Predicted and True Values')
+plt.xlabel('Score')
+plt.ylabel('Frequency')
+plt.legend()
+plt.grid(axis='y', linestyle='--', alpha=0.7)
+plt.show()

bert_no_ernie.py

@@ -4,7 +4,7 @@ import torch.nn as nn
 import torch.optim as optim
 from torch.utils.data import Dataset, DataLoader
 # scikit-learn Imports
-# from sklearn.metrics import accuracy_score, confusion_matrix
+from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
 from sklearn.model_selection import train_test_split
 # Bert imports
 from transformers import BertForSequenceClassification, AutoTokenizer
@@ -25,8 +25,8 @@ class SimpleHumorDataset(Dataset):
         super(SimpleHumorDataset,self).__init__()
         self.tokenizer = tokenizer
         self.max_length = max_length
-        self.text = dataframe['text'].to_list()
+        self.text = dataframe['text'].to_numpy()
-        self.labels = dataframe['is_humor'].to_list()
+        self.labels = dataframe['is_humor'].to_numpy()
     
     def __getitem__(self,idx:int):
         text = self.text[idx]    
@@ -52,41 +52,58 @@ class SimpleHumorDataset(Dataset):
         return len(self.labels)
 
 class CustomBert(nn.Module):
-    def __init__(self):
+    def __init__(self,dropout):
         super().__init__()
         #Bert + Custom Layers (Not a tuple any longer -- idk why)
         self.bfsc = BertForSequenceClassification.from_pretrained("bert-base-uncased")
+        self.dropout = nn.Dropout(dropout)
         self.classifier = nn.Linear(2,2)
-        self.sm = nn.Softmax(dim=1)
+        # self.sm = nn.Softmax(dim=1)
          
     def forward(self, input_ids, attention_mask):
         seq_out = self.bfsc(input_ids, attention_mask = attention_mask)
-        x = self.classifier(seq_out.logits)
+        return self.classifier(self.dropout(seq_out[0]))
-        return self.sm(x)
         
-def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader):
+    
+    def freeze_bert_params(self):
+        for param in self.bfsc.named_parameters():
+            param[1].requires_grad_(False)
+
+    def unfreeze_bert_params(self):
+        for param in self.bfsc.named_parameters():
+            param[1].requires_grad_(True)
+    
+def training_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,optimizer:optim.AdamW,train_loader:DataLoader,freeze_bert:bool):
     model.train()
-    total_loss = 0
+    if freeze_bert:
+        model.freeze_bert_params()
     
-    for train_batch in train_loader:
+    total_loss = 0
+    len_train_loader = len(train_loader)
+    for index,train_batch in enumerate(train_loader):
         # Set Gradient to Zero
         optimizer.zero_grad()
         # Unpack batch values and "push" it to GPU
         input_ids, att_mask, labels = train_batch.values()
+        # print(f"{input_ids.shape}, {att_mask.shape}, {labels.shape}")
+        # print(f"Iteration {index} of {len_train_loader}")
         input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE)
         # Feed Model with Data
         outputs = model(input_ids, attention_mask=att_mask)
+        # print(f"{model.bfsc.}")
+        # print(f"{outputs.shape}")
         loss = criterion(outputs,labels)
         loss.backward()
         optimizer.step()
         total_loss+=loss.item()
-    print(f"Total Loss is {(total_loss/len(train_loader)):.4f}")  
+    print(f"Training Loss is {(total_loss/len(train_loader)):.4f}")  
     return (total_loss/len(train_loader))       
 
 def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:DataLoader):
     model.eval()
     total, correct = 0.0, 0.0
     total_loss = 0.0
+    best_loss = 10.0
     with torch.no_grad():
         for val_batch in validation_loader:
             input_ids, att_mask ,labels = val_batch.values()
@@ -97,23 +114,50 @@ def eval_loop(model:CustomBert,criterion:nn.CrossEntropyLoss,validation_loader:D
             predictions = torch.argmax(outputs,1)
             total += labels.size(0)
             correct += (predictions == labels).sum().item()
-    print(f"Total Loss: {total_loss/len(validation_loader)} ### Test Accuracy {correct/total*100}%")
+            if total_loss/len(validation_loader) < best_loss:
+                best_loss = total_loss/len(validation_loader)
+                torch.save(model,"best_bert_model")
+    print(f"Validation Loss: {total_loss/len(validation_loader):.4f} ### Test Accuracy {correct/total*100:.4f}%")
     return total_loss/len(validation_loader)
 
+def test_loop(model:CustomBert, criterion:nn.CrossEntropyLoss, test_loader:DataLoader):
+    for batch in test_loader:
+        input_ids, att_mask, labels = batch.values()
+        input_ids, att_mask, labels = input_ids.to(DEVICE), att_mask.to(DEVICE), labels.to(DEVICE)
+        with torch.no_grad():
+            output = model(input_ids,att_mask)
+            output.detach().cpu().numpy()
+            labels.detach().cpu().numpy()
+            pred_flat = np.argmax(output,1).flatten()
+            print(accuracy_score(labels,pred_flat))
+
+def performance_metrics(true_labels,predictions):
+    confusion_matrix(true_labels,predictions)
+    accuracy_score(true_labels,predictions)
+    f1_score(true_labels,predictions)
+    pass
+
 
 if __name__ == "__main__":
-    torch.manual_seed(501)
+    
     # HYPERPARAMETERS
     # Set Max Epoch Amount
-    EPOCH = 1
+    EPOCH = 10
     # DROPOUT-PROBABILITY
     DROPOUT = 0.1
     # BATCHSIZE
-    BATCH_SIZE = 8
+    BATCH_SIZE = 16
     #LEARNING RATE
     LEARNING_RATE = 1e-5
+    # RANDOM SEED
+    RNDM_SEED = 501
+
+    torch.manual_seed(RNDM_SEED)
+    np.random.seed(RNDM_SEED)
+    torch.cuda.seed_all(RNDM_SEED)
+
     # Initialize Bert Model with dropout probability and Num End Layers
-    mybert = CustomBert()
+    mybert = CustomBert(DROPOUT)
     print("Bert Initialized")
     mybert.to(DEVICE)
 
@@ -122,27 +166,26 @@ if __name__ == "__main__":
     df = pd.read_csv("./data/hack.csv",encoding="latin1")
     print("Raw Data read")
 
-
     # Initialize BertTokenizer from Pretrained 
     tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased",do_lower_case=True)
     print("Tokenizer Initialized")
 
-    # print(tokenizer(df['text'][0],padding=True,truncation=True,max_length=256))
     #Split DataFrame into Train and Test Sets
     train,test = train_test_split(df,random_state=501,test_size=.2)
     print("Splitted Data in Train and Test Sets")
+    test,val = train_test_split(test,random_state=501,test_size=.5)
 
     # val = []
     # Create Custom Datasets for Train and Test
     train_data = SimpleHumorDataset(tokenizer,train)
-    # val_data = SimpleHumorDataset(tokenizer,val)
+    val_data = SimpleHumorDataset(tokenizer,val)
     test_data = SimpleHumorDataset(tokenizer,test)
     print("Custom Datasets created") 
 
 
    # Initialize Dataloader with Train and Test Sets
     train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
-    # val_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
+    validation_loader = DataLoader(dataset=val_data,batch_size=BATCH_SIZE,shuffle=True)
     test_loader = DataLoader(dataset=test_data,batch_size=BATCH_SIZE,shuffle=False)
     print("DataLoaders created")
 
@@ -153,20 +196,23 @@ if __name__ == "__main__":
     # Set Scheduler for dynamically Learning Rate adjustment
     loss_values = np.zeros(EPOCH)
     eval_values = np.zeros(EPOCH)
-    start = time.time()
+    freeze = False
-    for epoch in range(EPOCH):
     
+    for epoch in range(EPOCH):
+        start = time.time()
         print(f"For {epoch+1} the Scores are: ")
-        loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader)
+        loss_values[epoch] = training_loop(mybert,optimizer=optimizer_adamW,criterion=criterion_cross_entropy,train_loader=train_loader,freeze_bert=freeze)
         eval_values[epoch] = eval_loop(mybert,criterion=criterion_cross_entropy,validation_loader=test_loader)
-    end = time.time()
+        end = time.time()
-    print((end-start),"seconds per epoch needed")
+        print((end-start),"seconds per epoch needed")
     # Visualize Training Loss 
-    plt.plot(loss_values)
+    # plt.plot(loss_values)
-    plt.plot(eval_values)
+    # plt.plot(eval_values)
-    plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
+    # plt.hlines(np.mean(loss_values),xmin=0,xmax=EPOCH,colors='red',linestyles="dotted",label="Average Loss")
-    plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")
+    # plt.hlines(np.mean(eval_values),xmin=0,xmax=EPOCH,colors='green',linestyles="dashed",label="Average Val Loss")
-    plt.title("Test Loss")
+    # plt.title("Test Loss")
-    plt.xlabel("Num Epochs")
+    # plt.xlabel("Num Epochs")
-    plt.ylabel("Total Loss of Epoch")
+    # plt.ylabel("Total Loss of Epoch")
-    plt.show()
+    # plt.show()
+    for epoch in range(EPOCH):
+        test_loop(mybert,criterion_cross_entropy,validation_loader)