ANLP_WS24_CA2/LSTM.py

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from sklearn.metrics import mean_squared_error, r2_score
from torch.optim.lr_scheduler import ReduceLROnPlateau
import time
from tqdm import tqdm
from Datasets import GloveDataset as HumorDataset
import Datasets
import dataset_helper


class LSTMNetwork(nn.Module):
    def __init__(self, input_dim, hidden_dim, num_layers, output_dim, dropout=0.3):
        super(LSTMNetwork, self).__init__()
        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, dropout=dropout, batch_first=True)
        self.fc = nn.Linear(hidden_dim, output_dim)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        lstm_out, _ = self.lstm(x)
        # print(lstm_out)
        return self.fc(self.dropout(lstm_out))


def compute_metrics(predictions, labels):
    mse = mean_squared_error(labels, predictions)
    r2 = r2_score(labels, predictions)
    return mse, r2


def train_model(model, train_loader, val_loader, test_loader, epochs=10, device='cuda'):
    criterion = nn.MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=0.001)
    scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3, verbose=True)

    best_val_loss = float('inf')
    best_test_r2 = -float('inf')
    patience = 3
    counter = 0

    history = {'train_loss': [], 'val_loss': [], 'test_r2': [], 'test_mse': []}

    for epoch in range(epochs):
        model.train()
        total_loss = 0
        start_time = time.time()

        train_preds, train_labels = [], []

        for batch in tqdm(train_loader, desc=f"Epoch {epoch+1}/{epochs}", ncols=100):
            optimizer.zero_grad()
            inputs = batch[0].float().to(device)#batch['input_ids'].to(device)
            labels = batch[1].float().to(device)#batch['labels'].to(device)
            outputs = model(inputs)
            loss = criterion(outputs.squeeze(), labels)
            loss.backward()
            optimizer.step()
            total_loss += loss.item()
            train_preds.extend(outputs.squeeze().detach().cpu().numpy())
            train_labels.extend(labels.cpu().numpy())

        avg_train_loss = total_loss / len(train_loader)

        model.eval()
        val_loss = 0
        val_preds, val_labels = [], []

        with torch.no_grad():
            for batch in val_loader:
                inputs = batch[0].float().to(device)#batch['input_ids'].to(device)
                labels = batch[1].float().to(device)#batch['labels'].to(device)
                outputs = model(inputs)
                val_loss += criterion(outputs.squeeze(), labels).item()
                val_preds.extend(outputs.squeeze().cpu().numpy())
                val_labels.extend(labels.cpu().numpy())

        avg_val_loss = val_loss / len(val_loader)

        test_preds, test_labels = [], []

        with torch.no_grad():
            for batch in test_loader:
                inputs = batch[0].float().to(device)#batch['input_ids'].to(device)
                labels = batch[1].float().to(device)#batch['labels'].to(device)
                outputs = model(inputs)
                test_preds.extend(outputs.squeeze().cpu().numpy())
                test_labels.extend(labels.cpu().numpy())

        test_mse, test_r2 = compute_metrics(test_preds, test_labels)

        history['train_loss'].append(avg_train_loss)
        history['val_loss'].append(avg_val_loss)
        history['test_r2'].append(test_r2)
        history['test_mse'].append(test_mse)

        scheduler.step(avg_val_loss)

        epoch_time = time.time() - start_time
        print(f'Epoch {epoch+1}/{epochs} | Time: {epoch_time:.2f}s')
        print(f'Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f}')
        print(f'Test MSE: {test_mse:.4f} | Test R2: {test_r2:.4f}\n')

        if test_r2 > best_test_r2:
            best_test_r2 = test_r2
            torch.save(model.state_dict(), "best_lstm_model.pth")
            print(f"New best model saved (R2: {test_r2:.4f})")

        if avg_val_loss < best_val_loss:
            best_val_loss = avg_val_loss
            counter = 0
        else:
            counter += 1
            if counter >= patience:
                print("Early stopping triggered!")
                break

    return history


if __name__ == "__main__":
    input_dim = 128
    hidden_dim = 1024
    num_layers = 2
    output_dim = 1
    dropout = 0.2
    batch_size = 256
    epochs = 5
    
    DATA_PATH = "data/hack.csv"
    GLOVE_PATH = "data/glove.6b.100d.txt"
    EMBEDDING_DIM = 100
    TEST_SIZE = 0.1
    VAL_SIZE = 0.1
    params = {"max_len":128}
    # data_path = 'data/embedded_padded'
    embedding_matrix, word_index, vocab_size, d_model = dataset_helper.get_embedding_matrix(
        gloVe_path=GLOVE_PATH, emb_len=EMBEDDING_DIM)

    X, y = dataset_helper.load_preprocess_data(path_data=DATA_PATH, verbose=True)

    # Aufteilen der Daten
    data_split = dataset_helper.split_data(X, y, test_size=TEST_SIZE, val_size=VAL_SIZE)

    # Dataset und DataLoader
    train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=params["max_len"])
    val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=params["max_len"])
    test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=params["max_len"])


    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = LSTMNetwork(input_dim=input_dim, hidden_dim=hidden_dim, num_layers=num_layers, output_dim=output_dim, dropout=dropout).to(device)
    history = train_model(model, train_loader, val_loader, test_loader, epochs=epochs, device=device)