transformer mit bootstrap agg

transformer_bootstrap_agg.py

@@ -0,0 +1,310 @@
+import time
+import json
+import math
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+from nltk.tokenize import word_tokenize
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, Subset
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+
+from sklearn.metrics import accuracy_score, precision_recall_curve, f1_score, confusion_matrix, r2_score
+from sklearn.model_selection import KFold
+# local imports
+import ml_evaluation as ml_eval
+import ml_helper
+import ml_history
+import dataset_generator as data_gen
+# class imports
+import HumorDataset as humor_ds
+import EarlyStopping
+import BalancedCELoss
+
+
+torch.manual_seed(0)
+np.random.seed(0)
+
+
+best_model_filename = 'best_transformer_reg_model.pt'
+
+device = ml_helper.get_device(verbose=True)
+
+embedding_matrix, word_index, vocab_size, d_model = data_gen.create_embedding_matrix()
+
+vocab_size = len(embedding_matrix)
+d_model = len(embedding_matrix[0])
+vocab_size, d_model = embedding_matrix.size()
+print(f"vocab_size: {vocab_size}, d_model: {d_model}")
+
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, vocab_size=5000, dropout=0.1):
+        super().__init__()
+        self.dropout = nn.Dropout(p=dropout)
+
+        pe = torch.zeros(vocab_size, d_model)
+        position = torch.arange(0, vocab_size, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(
+            torch.arange(0, d_model, 2).float()
+            * (-math.log(10000.0) / d_model)
+        )
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0)
+        self.register_buffer("pe", pe)
+
+    def forward(self, x):
+        x = x + self.pe[:, : x.size(1), :]
+        return self.dropout(x)
+
+
+class TransformerBinaryClassifier(nn.Module):
+    def __init__(
+        self,
+        embeddings,
+        nhead=8,
+        dim_feedforward=2048,
+        num_layers=6,
+        positional_dropout=0.1,
+        classifier_dropout=0.1,
+        activation="relu",
+    ):
+        super().__init__()
+
+        vocab_size, d_model = embeddings.size()
+        assert d_model % nhead == 0, "nheads must divide evenly into d_model"
+
+        self.emb = nn.Embedding.from_pretrained(embeddings, freeze=False)
+
+        self.pos_encoder = PositionalEncoding(
+            d_model=d_model,
+            dropout=positional_dropout,
+            vocab_size=vocab_size,
+        )
+
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=d_model,
+            nhead=nhead,
+            dim_feedforward=dim_feedforward,
+            dropout=classifier_dropout,
+        )
+        self.transformer_encoder = nn.TransformerEncoder(
+            encoder_layer,
+            num_layers=num_layers,
+        )
+        self.batch_norm = nn.BatchNorm1d(d_model)
+        self.classifier = nn.Linear(d_model, 1)
+        self.d_model = d_model
+
+    def forward(self, x):
+        x = self.emb(x) * math.sqrt(self.d_model)
+        x = self.pos_encoder(x)
+        x = self.transformer_encoder(x)
+        x = x.mean(dim=1)
+        x = self.classifier(x)
+        return x
+
+
+def load_preprocess_data(path_data='data/hack.csv'):
+    df = pd.read_csv(path_data)
+    df = df.dropna(subset=['humor_rating'])
+
+    df['y'] = df['humor_rating']
+    X = df['text']
+    y = df['y']
+    return X, y
+
+
+X, y = load_preprocess_data()
+
+ret_dict = data_gen.split_data(X, y)
+
+params = {
+    'equalize_classes_loss_factor': 0.15,
+    'batch_size': 32,
+    'epochs': 2,
+    'lr': 1e-4,
+    'clipping_max_norm': 0,
+    'early_stopping_patience': 5,
+    'lr_scheduler_factor': 0.5,
+    'lr_scheduler_patience': 3,
+    'nhead': 2,
+    'num_layers': 3,
+    'hidden_dim': 10,
+    'positional_dropout': 0.5,
+    'classifier_dropout': 0.5,
+    'weight_decay': 1e-2
+}
+
+max_len = 280
+
+train_dataset = humor_ds.TextDataset(ret_dict['train']['X'], ret_dict['train']['y'], word_index, max_len=max_len)
+val_dataset = humor_ds.TextDataset(ret_dict['val']['X'], ret_dict['val']['y'], word_index, max_len=max_len)
+test_dataset = humor_ds.TextDataset(ret_dict['test']['X'], ret_dict['test']['y'], word_index, max_len=max_len)
+
+train_loader = DataLoader(train_dataset, batch_size=params['batch_size'], shuffle=True)
+val_loader = DataLoader(val_dataset, batch_size=params['batch_size'], shuffle=False)
+test_loader = DataLoader(test_dataset, batch_size=params['batch_size'], shuffle=False)
+
+early_stopping = EarlyStopping.EarlyStopping(patience=params['early_stopping_patience'], verbose=False)
+
+
+def train_model(model, train_dataset, criterion, optimizer, epochs, batch_size):
+    dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+    model.to(device)
+
+    # Store for plotting
+    train_losses, val_losses = [], []
+    train_r2_scores, val_r2_scores = [], []
+
+    for epoch in range(epochs):
+        model.train()
+        total_loss = 0
+        all_preds, all_targets = [], []
+
+        for inputs, targets in dataloader:
+            inputs, targets = inputs.to(device), targets.to(device)
+            optimizer.zero_grad()
+            outputs = model(inputs).squeeze()
+            loss = criterion(outputs, targets.float())
+            loss.backward()
+            optimizer.step()
+            total_loss += loss.item()
+
+            all_preds.extend(outputs.detach().cpu().numpy())
+            all_targets.extend(targets.detach().cpu().numpy())
+
+        # Calculate R2
+        r2 = r2_score(all_targets, all_preds)
+        train_losses.append(total_loss / len(dataloader))
+        train_r2_scores.append(r2)
+
+        # Validation phase
+        model.eval()
+        val_loss = 0
+        val_preds, val_targets = [], []
+
+        with torch.no_grad():
+            for inputs, targets in val_loader:
+                inputs, targets = inputs.to(device), targets.to(device)
+                outputs = model(inputs).squeeze()
+                loss = criterion(outputs, targets.float())
+                val_loss += loss.item()
+
+                val_preds.extend(outputs.cpu().numpy())
+                val_targets.extend(targets.cpu().numpy())
+
+        # Calculate Validation R2
+        val_r2 = r2_score(val_targets, val_preds)
+        val_losses.append(val_loss / len(val_loader))
+        val_r2_scores.append(val_r2)
+
+        print(f"Epoch {epoch + 1}/{epochs}, Loss: {total_loss / len(dataloader):.4f}, R^2 (Train): {r2:.4f}, Val R^2: {val_r2:.4f}")
+
+    return train_losses, val_losses, train_r2_scores, val_r2_scores
+
+
+def bootstrap_aggregation(ModelClass, train_dataset, num_models=5, epochs=10, batch_size=32, learning_rate=0.001):
+    models = []
+    all_train_losses, all_val_losses = [], []
+    all_train_r2_scores, all_val_r2_scores = [], []
+
+    subset_size = len(train_dataset) // num_models
+
+    for i in range(num_models):
+        print(f"Training Model {i + 1}/{num_models}...")
+        start_idx = i * subset_size
+        end_idx = start_idx + subset_size
+        subset_indices = list(range(0, start_idx)) + list(range(end_idx, len(train_dataset)))
+        subset = Subset(train_dataset, subset_indices)
+
+        model = ModelClass()
+        criterion = nn.MSELoss()
+        optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+        train_losses, val_losses, train_r2_scores, val_r2_scores = train_model(model, subset, criterion, optimizer, epochs, batch_size)
+        
+        models.append(model)
+        all_train_losses.append(train_losses)
+        all_val_losses.append(val_losses)
+        all_train_r2_scores.append(train_r2_scores)
+        all_val_r2_scores.append(val_r2_scores)
+
+    return models, all_train_losses, all_val_losses, all_train_r2_scores, all_val_r2_scores
+
+
+# Ensemble Prediction
+def ensemble_predict(models, test_dataset):
+    dataloader = DataLoader(test_dataset, batch_size=32, shuffle=False)
+    all_predictions = []
+
+    with torch.no_grad():
+        for inputs, _ in dataloader:
+            inputs = inputs.to(device)
+            predictions = torch.stack([model(inputs).squeeze() for model in models])
+            avg_predictions = predictions.mean(dim=0)
+            all_predictions.extend(avg_predictions.cpu().numpy())
+
+    return np.array(all_predictions)
+
+
+# Bootstrap Aggregating
+num_models = 2
+ensemble_models, all_train_losses, all_val_losses, all_train_r2_scores, all_val_r2_scores = bootstrap_aggregation(
+    lambda: TransformerBinaryClassifier(
+        embeddings=embedding_matrix,
+        nhead=params['nhead'],
+        num_layers=params['num_layers'],
+        dim_feedforward=params['hidden_dim'],
+        positional_dropout=params['positional_dropout'],
+        classifier_dropout=params['classifier_dropout']
+    ).to(device),
+    train_dataset,
+    num_models=num_models,
+    epochs=params['epochs'],
+    batch_size=params['batch_size'],
+    learning_rate=params['lr']
+)
+
+# Ensemble Prediction on Testset
+ensemble_predictions = ensemble_predict(ensemble_models, test_dataset)
+
+# Plotting
+fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6))
+
+# Plot Train and Validation Losses
+for i in range(num_models):
+    ax1.plot(range(1, params['epochs'] + 1), all_train_losses[i], label=f"Train Model {i+1}")
+    ax1.plot(range(1, params['epochs'] + 1), all_val_losses[i], label=f"Val Model {i+1}")
+
+ax1.set_title('Train and Validation Loss')
+ax1.set_xlabel('Epochs')
+ax1.set_ylabel('Loss')
+ax1.legend()
+
+# Plot Train and Validation R²
+for i in range(num_models):
+    ax2.plot(range(1, params['epochs'] + 1), all_train_r2_scores[i], label=f"Train Model {i+1}")
+    ax2.plot(range(1, params['epochs'] + 1), all_val_r2_scores[i], label=f"Val Model {i+1}")
+
+ax2.set_title('Train and Validation R²')
+ax2.set_xlabel('Epochs')
+ax2.set_ylabel('R²')
+ax2.legend()
+
+plt.tight_layout()
+plt.show()
+
+# Evaluation
+mse = mean_squared_error(test_dataset.labels.to_numpy(), ensemble_predictions)
+mae = mean_absolute_error(test_dataset.labels.to_numpy(), ensemble_predictions)
+r2 = r2_score(test_dataset.labels.to_numpy(), ensemble_predictions)
+
+print(f"Ensemble MSE: {mse:.4f}, MAE: {mae:.4f}, R²: {r2:.4f}")