diff --git a/BERT.py b/BERT.py index ebeeeb9..ae1fccc 100644 --- a/BERT.py +++ b/BERT.py @@ -9,6 +9,7 @@ from transformers import BertForSequenceClassification, AutoTokenizer import numpy as np from datetime import datetime import json +import itertools import Datasets import dataset_helper @@ -52,18 +53,26 @@ class CustomBert(nn.Module): if __name__ == '__main__': # Hyperparameter und Konfigurationen params = { - # Config - "max_len": 128, # Training - "epochs": 1, - "patience": 7, - "batch_size": 32, - "learning_rate": 1e-6, - "weight_decay": 5e-4 , + "epochs": [1], + "patience": [7], + "learning_rate": [1e-5, 1e-6], + "weight_decay": [5e-4], # Model - "dropout": 0.6 + "dropout": [0.6] } + # Generate permutations of hyperparameters + keys, values = zip(*params.items()) + grid_params = [dict(zip(keys, v)) for v in itertools.product(*values)] + best_params = {} + best_params_rmse = -1 + # Example usage of grid_params + # for param_set in grid_params: + # print(param_set) + print('Number of grid_params:', len(grid_params)) + + # Configs GLOVE_PATH = 'data/glove.6B.100d.txt' DATA_PATH = 'data/hack.csv' @@ -72,7 +81,11 @@ if __name__ == '__main__': TEST_SIZE = 0.1 VAL_SIZE = 0.1 - N_MODELS = 2 + MAX_LEN = 280 + BATCH_SIZE = 32 + + N_MODELS = 1 + USE_GIRD_SEARCH = True models = [] timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") @@ -91,17 +104,29 @@ if __name__ == '__main__': print("Tokenizer Initialized") # Dataset und DataLoader - train_dataset = Datasets.BertDataset(tokenizer, data_split['train']['X'], data_split['train']['y'], max_len=params["max_len"]) - val_dataset = Datasets.BertDataset(tokenizer, data_split['val']['X'], data_split['val']['y'], max_len=params["max_len"]) - test_dataset = Datasets.BertDataset(tokenizer, data_split['test']['X'], data_split['test']['y'], max_len=params["max_len"]) + train_dataset = Datasets.BertDataset(tokenizer, data_split['train']['X'], data_split['train']['y'], max_len=MAX_LEN) + val_dataset = Datasets.BertDataset(tokenizer, data_split['val']['X'], data_split['val']['y'], max_len=MAX_LEN) + test_dataset = Datasets.BertDataset(tokenizer, data_split['test']['X'], data_split['test']['y'], max_len=MAX_LEN) - train_loader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=params["batch_size"], shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=params["batch_size"], shuffle=False) + train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False) + test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False) subset_size = len(train_dataset) // N_MODELS device = ml_helper.get_device(verbose=True, include_mps=False) + # assert if N_MODLES > 1, than grid_params should be len 1 + if N_MODELS > 1 and len(grid_params) > 1 or N_MODELS > 1 and USE_GIRD_SEARCH: + raise ValueError("If N_MODELS > 1, than grid_params should be len 1") + + if not USE_GIRD_SEARCH: + print('Using best params') + # load best params + params_name = f'models/best_params_BERT.json' + with open(params_name, 'r') as f: + best_params = json.load(f) + grid_params = [best_params] + for i in range(N_MODELS): model_name = f'BERT.pt' hist_name = f'BERT_history' @@ -112,46 +137,60 @@ if __name__ == '__main__': subset_indices = dataset_helper.ensemble_data_idx(train_dataset.labels, N_MODELS, i, methods='bootstrap') train_dataset_sub = Subset(train_dataset, subset_indices) - train_loader = DataLoader(train_dataset_sub, batch_size=params["batch_size"], shuffle=True) + train_loader = DataLoader(train_dataset_sub, batch_size=BATCH_SIZE, shuffle=True) + + for para_idx, params in enumerate(grid_params): + if len(grid_params) > 1: + model_name = f'BERT_{i}_param_{para_idx}.pt' + hist_name = f'BERT_{i}_param_{para_idx}_history' - model = CustomBert(dropout=params["dropout"]) - model = model.to(device) + model = CustomBert(dropout=params["dropout"]) + model = model.to(device) - criterion = nn.MSELoss() - optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"]) - early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name) + criterion = nn.MSELoss() + optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"]) + early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name) - hist = ml_history.History() + hist = ml_history.History() - # Training und Validierung - for epoch in range(params["epochs"]): - ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"], bert_freeze=FREEZE_BERT, is_bert=True) + # Training und Validierung + for epoch in range(params["epochs"]): + ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"], bert_freeze=FREEZE_BERT, is_bert=True) - val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist, is_bert=True) + val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist, is_bert=True) - early_stopping(val_rmse, model) - if early_stopping.early_stop: - print("Early stopping triggered.") - break + early_stopping(val_rmse, model) + if early_stopping.early_stop: + print("Early stopping triggered.") + break - # Load best model - model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False)) - models.append(model) + # Load best model + model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False)) + models.append(model) - # Test Evaluation - test_labels, test_preds = ml_train.test_loop(model, test_loader, device, is_bert=True) + # Test Evaluation + test_labels, test_preds = ml_train.test_loop(model, test_loader, device, is_bert=True) - hist.add_test_results(test_labels, test_preds) + hist.add_test_results(test_labels, test_preds) - # save training history - hist.save_history(hist_name, timestamp) + # save training history + hist.save_history(hist_name, timestamp) - # RMSE, MAE und R²-Score für das Test-Set - test_mae = mean_absolute_error(test_labels, test_preds) - test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds)) - test_r2 = r2_score(test_labels, test_preds) - print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}") + # RMSE, MAE und R²-Score für das Test-Set + test_mae = mean_absolute_error(test_labels, test_preds) + test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds)) + test_r2 = r2_score(test_labels, test_preds) + print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}") + + if test_rmse > best_params_rmse: + best_params_rmse = test_rmse + best_params = params + + if len(grid_params) > 1: + best_params_name = f'models/best_params_BERT.json' + with open(best_params_name, 'w') as f: + json.dump(best_params, f) if N_MODELS >1: diff --git a/Transformer.py b/Transformer.py index 41c8a49..1568cd0 100644 --- a/Transformer.py +++ b/Transformer.py @@ -9,6 +9,7 @@ from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score import numpy as np from datetime import datetime import json +import itertools import Datasets import dataset_helper @@ -108,21 +109,28 @@ class TransformerBinaryClassifier(nn.Module): if __name__ == '__main__': # Hyperparameter und Konfigurationen params = { - # Config - "max_len": 280, # Training - "epochs": 1, - "patience": 7, - "batch_size": 32, - "learning_rate": 1e-4, # 1e-4 - "weight_decay": 5e-4 , + "epochs": [1], + "patience": [7], + "learning_rate": [1e-4], # 1e-4 + "weight_decay": [5e-4], # Model - 'nhead': 2, # 5 - "dropout": 0.2, - 'hiden_dim': 2048, - 'num_layers': 6 + 'nhead': [2], # 5 + "dropout": [0.2], + 'hiden_dim': [1024, 2048], + 'num_layers': [6] } + # Generate permutations of hyperparameters + keys, values = zip(*params.items()) + grid_params = [dict(zip(keys, v)) for v in itertools.product(*values)] + best_params = {} + best_params_rmse = -1 + # Example usage of grid_params + # for param_set in grid_params: + # print(param_set) + print('Number of grid_params:', len(grid_params)) + # Configs GLOVE_PATH = 'data/glove.6B.100d.txt' DATA_PATH = 'data/hack.csv' @@ -130,7 +138,11 @@ if __name__ == '__main__': TEST_SIZE = 0.1 VAL_SIZE = 0.1 - N_MODELS = 2 + MAX_LEN = 280 + BATCH_SIZE = 32 + + N_MODELS = 1 + USE_GIRD_SEARCH = True models = [] timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") @@ -145,17 +157,29 @@ if __name__ == '__main__': data_split = dataset_helper.split_data(X, y, test_size=TEST_SIZE, val_size=VAL_SIZE) # Dataset und DataLoader - train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=params["max_len"]) - val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=params["max_len"]) - test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=params["max_len"]) + train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=MAX_LEN) + val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=MAX_LEN) + test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=MAX_LEN) - train_loader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=params["batch_size"], shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=params["batch_size"], shuffle=False) + train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False) + test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False) subset_size = len(train_dataset) // N_MODELS device = ml_helper.get_device(verbose=True, include_mps=False) + # assert if N_MODLES > 1, than grid_params should be len 1 + if N_MODELS > 1 and len(grid_params) > 1 or N_MODELS > 1 and USE_GIRD_SEARCH: + raise ValueError("If N_MODELS > 1, than grid_params should be len 1") + + if not USE_GIRD_SEARCH: + print('Using best params') + # load best params + params_name = f'models/best_params_Transformer.json' + with open(params_name, 'r') as f: + best_params = json.load(f) + grid_params = [best_params] + for i in range(N_MODELS): model_name = f'Transformer.pt' hist_name = f'Transformer_history' @@ -166,53 +190,67 @@ if __name__ == '__main__': subset_indices = dataset_helper.ensemble_data_idx(train_dataset.labels, N_MODELS, i, methods='bootstrap') train_dataset_sub = Subset(train_dataset, subset_indices) - train_loader = DataLoader(train_dataset_sub, batch_size=params["batch_size"], shuffle=True) + train_loader = DataLoader(train_dataset_sub, batch_size=BATCH_SIZE, shuffle=True) - # Modell initialisieren - model = TransformerBinaryClassifier( - embeddings=embedding_matrix, - nhead=params['nhead'], - dim_feedforward=params['hiden_dim'], - num_layers=params['num_layers'], - positional_dropout=params["dropout"], - classifier_dropout=params["dropout"], - ) - model = model.to(device) + for para_idx, params in enumerate(grid_params): + if len(grid_params) > 1: + model_name = f'Transformer_{i}_param_{para_idx}.pt' + hist_name = f'Transformer_{i}_param_{para_idx}_history' - criterion = nn.MSELoss() - optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"]) #, weight_decay=params["weight_decay"]) - early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name) + # Modell initialisieren + model = TransformerBinaryClassifier( + embeddings=embedding_matrix, + nhead=params['nhead'], + dim_feedforward=params['hiden_dim'], + num_layers=params['num_layers'], + positional_dropout=params["dropout"], + classifier_dropout=params["dropout"], + ) + model = model.to(device) - hist = ml_history.History() + criterion = nn.MSELoss() + optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"]) #, weight_decay=params["weight_decay"]) + early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name) - # Training und Validierung - for epoch in range(params["epochs"]): - ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"]) + hist = ml_history.History() - val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist) + # Training und Validierung + for epoch in range(params["epochs"]): + ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"]) - early_stopping(val_rmse, model) - if early_stopping.early_stop: - print("Early stopping triggered.") - break + val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist) - # Load best model - model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False)) - models.append(model) - - # Test Evaluation - test_labels, test_preds = ml_train.test_loop(model, test_loader, device) + early_stopping(val_rmse, model) + if early_stopping.early_stop: + print("Early stopping triggered.") + break - hist.add_test_results(test_labels, test_preds) + # Load best model + model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False)) + models.append(model) + + # Test Evaluation + test_labels, test_preds = ml_train.test_loop(model, test_loader, device) - # save training history - hist.save_history(hist_name, timestamp) + hist.add_test_results(test_labels, test_preds) - # RMSE, MAE und R²-Score für das Test-Set - test_mae = mean_absolute_error(test_labels, test_preds) - test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds)) - test_r2 = r2_score(test_labels, test_preds) - print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}") + # save training history + hist.save_history(hist_name, timestamp) + + # RMSE, MAE und R²-Score für das Test-Set + test_mae = mean_absolute_error(test_labels, test_preds) + test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds)) + test_r2 = r2_score(test_labels, test_preds) + print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}") + + if test_rmse > best_params_rmse: + best_params_rmse = test_rmse + best_params = params + + if len(grid_params) > 1: + best_params_name = f'models/best_params_Transformer.json' + with open(best_params_name, 'w') as f: + json.dump(best_params, f) if N_MODELS >1: # Ensemble Prediction