diff --git a/CNN.py b/CNN.py index 52d56f1..55fb68b 100644 --- a/CNN.py +++ b/CNN.py @@ -8,6 +8,7 @@ from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score import numpy as np from datetime import datetime import json +import itertools import Datasets import dataset_helper @@ -34,7 +35,7 @@ class EnhancedCNNRegressor(nn.Module): nn.Conv2d(1, num_filters, (fs, embedding_dim)), nn.BatchNorm2d(num_filters), # Batch-Normalisierung nn.ReLU(), - nn.MaxPool2d((params["max_len"] - fs + 1, 1)), + nn.MaxPool2d((MAX_LEN - fs + 1, 1)), nn.Dropout(dropout) # Dropout nach jeder Schicht ) for fs in filter_sizes @@ -56,20 +57,27 @@ class EnhancedCNNRegressor(nn.Module): if __name__ == '__main__': # Hyperparameter und Konfigurationen params = { - # Config - "max_len": 280, # Training - "epochs": 5, - "patience": 7, - "batch_size": 32, - "learning_rate": 0.001, - "weight_decay": 5e-4 , + "epochs": [5], + "patience": [7], + "learning_rate": [0.001], + "weight_decay": [5e-4] , # Model - "filter_sizes": [2, 3, 4, 5], - "num_filters": 150, - "dropout": 0.6 + "filter_sizes": [[2, 3, 4, 5]], + "num_filters": [150], + "dropout": [0.6] } + # Generate permutations of hyperparameters + keys, values = zip(*params.items()) + grid_params = [dict(zip(keys, v)) for v in itertools.product(*values)] + best_params = {} + best_params_rmse = -1 + # Example usage of grid_params + # for param_set in grid_params: + # print(param_set) + print('Number of grid_params:', len(grid_params)) + # Configs GLOVE_PATH = 'data/glove.6B.100d.txt' DATA_PATH = 'data/hack.csv' @@ -77,7 +85,11 @@ if __name__ == '__main__': TEST_SIZE = 0.1 VAL_SIZE = 0.1 - N_MODELS = 1 + MAX_LEN = 280 + BATCH_SIZE = 32 + + N_MODELS = 2 + USE_GIRD_SEARCH = False models = [] timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") @@ -92,17 +104,30 @@ if __name__ == '__main__': data_split = dataset_helper.split_data(X, y, test_size=TEST_SIZE, val_size=VAL_SIZE) # Dataset und DataLoader - train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=params["max_len"]) - val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=params["max_len"]) - test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=params["max_len"]) + train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=MAX_LEN) + val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=MAX_LEN) + test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=MAX_LEN) - train_loader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=params["batch_size"], shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=params["batch_size"], shuffle=False) + train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False) + test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False) subset_size = len(train_dataset) // N_MODELS device = ml_helper.get_device(verbose=True, include_mps=False) + # assert if N_MODLES > 1, than grid_params should be len 1 + if N_MODELS > 1 and len(grid_params) > 1 or N_MODELS > 1 and USE_GIRD_SEARCH: + raise ValueError("If N_MODELS > 1, than grid_params should be len 1") + + if not USE_GIRD_SEARCH: + print('Using best params') + # load best params + params_name = f'models/best_params_CNN.json' + with open(params_name, 'r') as f: + best_params = json.load(f) + grid_params = [best_params] + + for i in range(N_MODELS): model_name = f'CNN.pt' hist_name = f'CNN_history' @@ -113,52 +138,66 @@ if __name__ == '__main__': subset_indices = dataset_helper.ensemble_data_idx(train_dataset.labels, N_MODELS, i, methods='bootstrap') train_dataset_sub = Subset(train_dataset, subset_indices) - train_loader = DataLoader(train_dataset_sub, batch_size=params["batch_size"], shuffle=True) + train_loader = DataLoader(train_dataset_sub, batch_size=BATCH_SIZE, shuffle=True) - model = EnhancedCNNRegressor( - vocab_size=vocab_size, - embedding_dim=EMBEDDING_DIM, - filter_sizes=params["filter_sizes"], - num_filters=params["num_filters"], - embedding_matrix=embedding_matrix, - dropout=params["dropout"] - ) - model = model.to(device) - - criterion = nn.MSELoss() - optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"]) - early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name) + for para_idx, params in enumerate(grid_params): + if len(grid_params) > 1: + model_name = f'CNN_{i}_param_{para_idx}.pt' + hist_name = f'CNN_{i}_param_{para_idx}_history' - hist = ml_history.History() + model = EnhancedCNNRegressor( + vocab_size=vocab_size, + embedding_dim=EMBEDDING_DIM, + filter_sizes=params["filter_sizes"], + num_filters=params["num_filters"], + embedding_matrix=embedding_matrix, + dropout=params["dropout"] + ) + model = model.to(device) + + criterion = nn.MSELoss() + optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"]) + early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name) - # Training und Validierung - for epoch in range(params["epochs"]): - ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"]) + hist = ml_history.History() - val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist) + # Training und Validierung + for epoch in range(params["epochs"]): + ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"]) - early_stopping(val_rmse, model) - if early_stopping.early_stop: - print("Early stopping triggered.") - break + val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist) - # Load best model - model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False)) - models.append(model) + early_stopping(val_rmse, model) + if early_stopping.early_stop: + print("Early stopping triggered.") + break - # Test Evaluation - test_labels, test_preds = ml_train.test_loop(model, test_loader, device) + # Load best model + model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False)) + models.append(model) - hist.add_test_results(test_labels, test_preds) + # Test Evaluation + test_labels, test_preds = ml_train.test_loop(model, test_loader, device) - # save training history - hist.save_history(hist_name, timestamp) + hist.add_test_results(test_labels, test_preds) - # RMSE, MAE und R²-Score für das Test-Set - test_mae = mean_absolute_error(test_labels, test_preds) - test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds)) - test_r2 = r2_score(test_labels, test_preds) - print(f"Model: {model_name} Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}") + # save training history + hist.save_history(hist_name, timestamp) + + # RMSE, MAE und R²-Score für das Test-Set + test_mae = mean_absolute_error(test_labels, test_preds) + test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds)) + test_r2 = r2_score(test_labels, test_preds) + print(f"Model: {model_name} Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}") + + if test_rmse > best_params_rmse: + best_params_rmse = test_rmse + best_params = params + + if len(grid_params) > 1: + best_params_name = f'models/best_params_CNN.json' + with open(best_params_name, 'w') as f: + json.dump(best_params, f) if N_MODELS >1: # Ensemble Prediction