added grid search
parent
e9e2bf1b8a
commit
3ad2d37ea2
125
BERT.py
125
BERT.py
|
|
@ -9,6 +9,7 @@ from transformers import BertForSequenceClassification, AutoTokenizer
|
|||
import numpy as np
|
||||
from datetime import datetime
|
||||
import json
|
||||
import itertools
|
||||
|
||||
import Datasets
|
||||
import dataset_helper
|
||||
|
|
@ -52,18 +53,26 @@ class CustomBert(nn.Module):
|
|||
if __name__ == '__main__':
|
||||
# Hyperparameter und Konfigurationen
|
||||
params = {
|
||||
# Config
|
||||
"max_len": 128,
|
||||
# Training
|
||||
"epochs": 1,
|
||||
"patience": 7,
|
||||
"batch_size": 32,
|
||||
"learning_rate": 1e-6,
|
||||
"weight_decay": 5e-4 ,
|
||||
"epochs": [1],
|
||||
"patience": [7],
|
||||
"learning_rate": [1e-5, 1e-6],
|
||||
"weight_decay": [5e-4],
|
||||
# Model
|
||||
"dropout": 0.6
|
||||
"dropout": [0.6]
|
||||
}
|
||||
|
||||
# Generate permutations of hyperparameters
|
||||
keys, values = zip(*params.items())
|
||||
grid_params = [dict(zip(keys, v)) for v in itertools.product(*values)]
|
||||
best_params = {}
|
||||
best_params_rmse = -1
|
||||
# Example usage of grid_params
|
||||
# for param_set in grid_params:
|
||||
# print(param_set)
|
||||
print('Number of grid_params:', len(grid_params))
|
||||
|
||||
|
||||
# Configs
|
||||
GLOVE_PATH = 'data/glove.6B.100d.txt'
|
||||
DATA_PATH = 'data/hack.csv'
|
||||
|
|
@ -72,7 +81,11 @@ if __name__ == '__main__':
|
|||
TEST_SIZE = 0.1
|
||||
VAL_SIZE = 0.1
|
||||
|
||||
N_MODELS = 2
|
||||
MAX_LEN = 280
|
||||
BATCH_SIZE = 32
|
||||
|
||||
N_MODELS = 1
|
||||
USE_GIRD_SEARCH = True
|
||||
|
||||
models = []
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
|
|
@ -91,17 +104,29 @@ if __name__ == '__main__':
|
|||
print("Tokenizer Initialized")
|
||||
|
||||
# Dataset und DataLoader
|
||||
train_dataset = Datasets.BertDataset(tokenizer, data_split['train']['X'], data_split['train']['y'], max_len=params["max_len"])
|
||||
val_dataset = Datasets.BertDataset(tokenizer, data_split['val']['X'], data_split['val']['y'], max_len=params["max_len"])
|
||||
test_dataset = Datasets.BertDataset(tokenizer, data_split['test']['X'], data_split['test']['y'], max_len=params["max_len"])
|
||||
train_dataset = Datasets.BertDataset(tokenizer, data_split['train']['X'], data_split['train']['y'], max_len=MAX_LEN)
|
||||
val_dataset = Datasets.BertDataset(tokenizer, data_split['val']['X'], data_split['val']['y'], max_len=MAX_LEN)
|
||||
test_dataset = Datasets.BertDataset(tokenizer, data_split['test']['X'], data_split['test']['y'], max_len=MAX_LEN)
|
||||
|
||||
train_loader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True)
|
||||
val_loader = DataLoader(val_dataset, batch_size=params["batch_size"], shuffle=False)
|
||||
test_loader = DataLoader(test_dataset, batch_size=params["batch_size"], shuffle=False)
|
||||
train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
|
||||
val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)
|
||||
test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)
|
||||
|
||||
subset_size = len(train_dataset) // N_MODELS
|
||||
device = ml_helper.get_device(verbose=True, include_mps=False)
|
||||
|
||||
# assert if N_MODLES > 1, than grid_params should be len 1
|
||||
if N_MODELS > 1 and len(grid_params) > 1 or N_MODELS > 1 and USE_GIRD_SEARCH:
|
||||
raise ValueError("If N_MODELS > 1, than grid_params should be len 1")
|
||||
|
||||
if not USE_GIRD_SEARCH:
|
||||
print('Using best params')
|
||||
# load best params
|
||||
params_name = f'models/best_params_BERT.json'
|
||||
with open(params_name, 'r') as f:
|
||||
best_params = json.load(f)
|
||||
grid_params = [best_params]
|
||||
|
||||
for i in range(N_MODELS):
|
||||
model_name = f'BERT.pt'
|
||||
hist_name = f'BERT_history'
|
||||
|
|
@ -112,46 +137,60 @@ if __name__ == '__main__':
|
|||
|
||||
subset_indices = dataset_helper.ensemble_data_idx(train_dataset.labels, N_MODELS, i, methods='bootstrap')
|
||||
train_dataset_sub = Subset(train_dataset, subset_indices)
|
||||
train_loader = DataLoader(train_dataset_sub, batch_size=params["batch_size"], shuffle=True)
|
||||
train_loader = DataLoader(train_dataset_sub, batch_size=BATCH_SIZE, shuffle=True)
|
||||
|
||||
for para_idx, params in enumerate(grid_params):
|
||||
if len(grid_params) > 1:
|
||||
model_name = f'BERT_{i}_param_{para_idx}.pt'
|
||||
hist_name = f'BERT_{i}_param_{para_idx}_history'
|
||||
|
||||
|
||||
model = CustomBert(dropout=params["dropout"])
|
||||
model = model.to(device)
|
||||
model = CustomBert(dropout=params["dropout"])
|
||||
model = model.to(device)
|
||||
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"])
|
||||
early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name)
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"], weight_decay=params["weight_decay"])
|
||||
early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name)
|
||||
|
||||
hist = ml_history.History()
|
||||
hist = ml_history.History()
|
||||
|
||||
# Training und Validierung
|
||||
for epoch in range(params["epochs"]):
|
||||
ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"], bert_freeze=FREEZE_BERT, is_bert=True)
|
||||
# Training und Validierung
|
||||
for epoch in range(params["epochs"]):
|
||||
ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"], bert_freeze=FREEZE_BERT, is_bert=True)
|
||||
|
||||
val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist, is_bert=True)
|
||||
val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist, is_bert=True)
|
||||
|
||||
early_stopping(val_rmse, model)
|
||||
if early_stopping.early_stop:
|
||||
print("Early stopping triggered.")
|
||||
break
|
||||
early_stopping(val_rmse, model)
|
||||
if early_stopping.early_stop:
|
||||
print("Early stopping triggered.")
|
||||
break
|
||||
|
||||
# Load best model
|
||||
model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False))
|
||||
models.append(model)
|
||||
# Load best model
|
||||
model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False))
|
||||
models.append(model)
|
||||
|
||||
# Test Evaluation
|
||||
test_labels, test_preds = ml_train.test_loop(model, test_loader, device, is_bert=True)
|
||||
# Test Evaluation
|
||||
test_labels, test_preds = ml_train.test_loop(model, test_loader, device, is_bert=True)
|
||||
|
||||
hist.add_test_results(test_labels, test_preds)
|
||||
hist.add_test_results(test_labels, test_preds)
|
||||
|
||||
# save training history
|
||||
hist.save_history(hist_name, timestamp)
|
||||
# save training history
|
||||
hist.save_history(hist_name, timestamp)
|
||||
|
||||
# RMSE, MAE und R²-Score für das Test-Set
|
||||
test_mae = mean_absolute_error(test_labels, test_preds)
|
||||
test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds))
|
||||
test_r2 = r2_score(test_labels, test_preds)
|
||||
print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}")
|
||||
# RMSE, MAE und R²-Score für das Test-Set
|
||||
test_mae = mean_absolute_error(test_labels, test_preds)
|
||||
test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds))
|
||||
test_r2 = r2_score(test_labels, test_preds)
|
||||
print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}")
|
||||
|
||||
if test_rmse > best_params_rmse:
|
||||
best_params_rmse = test_rmse
|
||||
best_params = params
|
||||
|
||||
if len(grid_params) > 1:
|
||||
best_params_name = f'models/best_params_BERT.json'
|
||||
with open(best_params_name, 'w') as f:
|
||||
json.dump(best_params, f)
|
||||
|
||||
|
||||
if N_MODELS >1:
|
||||
|
|
|
|||
148
Transformer.py
148
Transformer.py
|
|
@ -9,6 +9,7 @@ from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
|
|||
import numpy as np
|
||||
from datetime import datetime
|
||||
import json
|
||||
import itertools
|
||||
|
||||
import Datasets
|
||||
import dataset_helper
|
||||
|
|
@ -108,21 +109,28 @@ class TransformerBinaryClassifier(nn.Module):
|
|||
if __name__ == '__main__':
|
||||
# Hyperparameter und Konfigurationen
|
||||
params = {
|
||||
# Config
|
||||
"max_len": 280,
|
||||
# Training
|
||||
"epochs": 1,
|
||||
"patience": 7,
|
||||
"batch_size": 32,
|
||||
"learning_rate": 1e-4, # 1e-4
|
||||
"weight_decay": 5e-4 ,
|
||||
"epochs": [1],
|
||||
"patience": [7],
|
||||
"learning_rate": [1e-4], # 1e-4
|
||||
"weight_decay": [5e-4],
|
||||
# Model
|
||||
'nhead': 2, # 5
|
||||
"dropout": 0.2,
|
||||
'hiden_dim': 2048,
|
||||
'num_layers': 6
|
||||
'nhead': [2], # 5
|
||||
"dropout": [0.2],
|
||||
'hiden_dim': [1024, 2048],
|
||||
'num_layers': [6]
|
||||
}
|
||||
|
||||
# Generate permutations of hyperparameters
|
||||
keys, values = zip(*params.items())
|
||||
grid_params = [dict(zip(keys, v)) for v in itertools.product(*values)]
|
||||
best_params = {}
|
||||
best_params_rmse = -1
|
||||
# Example usage of grid_params
|
||||
# for param_set in grid_params:
|
||||
# print(param_set)
|
||||
print('Number of grid_params:', len(grid_params))
|
||||
|
||||
# Configs
|
||||
GLOVE_PATH = 'data/glove.6B.100d.txt'
|
||||
DATA_PATH = 'data/hack.csv'
|
||||
|
|
@ -130,7 +138,11 @@ if __name__ == '__main__':
|
|||
TEST_SIZE = 0.1
|
||||
VAL_SIZE = 0.1
|
||||
|
||||
N_MODELS = 2
|
||||
MAX_LEN = 280
|
||||
BATCH_SIZE = 32
|
||||
|
||||
N_MODELS = 1
|
||||
USE_GIRD_SEARCH = True
|
||||
|
||||
models = []
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
|
|
@ -145,17 +157,29 @@ if __name__ == '__main__':
|
|||
data_split = dataset_helper.split_data(X, y, test_size=TEST_SIZE, val_size=VAL_SIZE)
|
||||
|
||||
# Dataset und DataLoader
|
||||
train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=params["max_len"])
|
||||
val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=params["max_len"])
|
||||
test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=params["max_len"])
|
||||
train_dataset = Datasets.GloveDataset(data_split['train']['X'], data_split['train']['y'], word_index, max_len=MAX_LEN)
|
||||
val_dataset = Datasets.GloveDataset(data_split['val']['X'], data_split['val']['y'], word_index, max_len=MAX_LEN)
|
||||
test_dataset = Datasets.GloveDataset(data_split['test']['X'], data_split['test']['y'], word_index, max_len=MAX_LEN)
|
||||
|
||||
train_loader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True)
|
||||
val_loader = DataLoader(val_dataset, batch_size=params["batch_size"], shuffle=False)
|
||||
test_loader = DataLoader(test_dataset, batch_size=params["batch_size"], shuffle=False)
|
||||
train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
|
||||
val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)
|
||||
test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)
|
||||
|
||||
subset_size = len(train_dataset) // N_MODELS
|
||||
device = ml_helper.get_device(verbose=True, include_mps=False)
|
||||
|
||||
# assert if N_MODLES > 1, than grid_params should be len 1
|
||||
if N_MODELS > 1 and len(grid_params) > 1 or N_MODELS > 1 and USE_GIRD_SEARCH:
|
||||
raise ValueError("If N_MODELS > 1, than grid_params should be len 1")
|
||||
|
||||
if not USE_GIRD_SEARCH:
|
||||
print('Using best params')
|
||||
# load best params
|
||||
params_name = f'models/best_params_Transformer.json'
|
||||
with open(params_name, 'r') as f:
|
||||
best_params = json.load(f)
|
||||
grid_params = [best_params]
|
||||
|
||||
for i in range(N_MODELS):
|
||||
model_name = f'Transformer.pt'
|
||||
hist_name = f'Transformer_history'
|
||||
|
|
@ -166,53 +190,67 @@ if __name__ == '__main__':
|
|||
|
||||
subset_indices = dataset_helper.ensemble_data_idx(train_dataset.labels, N_MODELS, i, methods='bootstrap')
|
||||
train_dataset_sub = Subset(train_dataset, subset_indices)
|
||||
train_loader = DataLoader(train_dataset_sub, batch_size=params["batch_size"], shuffle=True)
|
||||
train_loader = DataLoader(train_dataset_sub, batch_size=BATCH_SIZE, shuffle=True)
|
||||
|
||||
# Modell initialisieren
|
||||
model = TransformerBinaryClassifier(
|
||||
embeddings=embedding_matrix,
|
||||
nhead=params['nhead'],
|
||||
dim_feedforward=params['hiden_dim'],
|
||||
num_layers=params['num_layers'],
|
||||
positional_dropout=params["dropout"],
|
||||
classifier_dropout=params["dropout"],
|
||||
)
|
||||
model = model.to(device)
|
||||
for para_idx, params in enumerate(grid_params):
|
||||
if len(grid_params) > 1:
|
||||
model_name = f'Transformer_{i}_param_{para_idx}.pt'
|
||||
hist_name = f'Transformer_{i}_param_{para_idx}_history'
|
||||
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"]) #, weight_decay=params["weight_decay"])
|
||||
early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name)
|
||||
# Modell initialisieren
|
||||
model = TransformerBinaryClassifier(
|
||||
embeddings=embedding_matrix,
|
||||
nhead=params['nhead'],
|
||||
dim_feedforward=params['hiden_dim'],
|
||||
num_layers=params['num_layers'],
|
||||
positional_dropout=params["dropout"],
|
||||
classifier_dropout=params["dropout"],
|
||||
)
|
||||
model = model.to(device)
|
||||
|
||||
hist = ml_history.History()
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=params["learning_rate"]) #, weight_decay=params["weight_decay"])
|
||||
early_stopping = EarlyStopping.EarlyStoppingCallback(patience=params["patience"], verbose=True, model_name=model_name)
|
||||
|
||||
# Training und Validierung
|
||||
for epoch in range(params["epochs"]):
|
||||
ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"])
|
||||
hist = ml_history.History()
|
||||
|
||||
val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist)
|
||||
# Training und Validierung
|
||||
for epoch in range(params["epochs"]):
|
||||
ml_train.train_epoch(model, train_loader, criterion, optimizer, device, hist, epoch, params["epochs"])
|
||||
|
||||
early_stopping(val_rmse, model)
|
||||
if early_stopping.early_stop:
|
||||
print("Early stopping triggered.")
|
||||
break
|
||||
val_rmse = ml_train.validate_epoch(model, val_loader, epoch, criterion, device, hist)
|
||||
|
||||
# Load best model
|
||||
model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False))
|
||||
models.append(model)
|
||||
|
||||
# Test Evaluation
|
||||
test_labels, test_preds = ml_train.test_loop(model, test_loader, device)
|
||||
early_stopping(val_rmse, model)
|
||||
if early_stopping.early_stop:
|
||||
print("Early stopping triggered.")
|
||||
break
|
||||
|
||||
hist.add_test_results(test_labels, test_preds)
|
||||
# Load best model
|
||||
model.load_state_dict(torch.load('models/checkpoints/' + model_name, weights_only=False))
|
||||
models.append(model)
|
||||
|
||||
# Test Evaluation
|
||||
test_labels, test_preds = ml_train.test_loop(model, test_loader, device)
|
||||
|
||||
# save training history
|
||||
hist.save_history(hist_name, timestamp)
|
||||
hist.add_test_results(test_labels, test_preds)
|
||||
|
||||
# RMSE, MAE und R²-Score für das Test-Set
|
||||
test_mae = mean_absolute_error(test_labels, test_preds)
|
||||
test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds))
|
||||
test_r2 = r2_score(test_labels, test_preds)
|
||||
print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}")
|
||||
# save training history
|
||||
hist.save_history(hist_name, timestamp)
|
||||
|
||||
# RMSE, MAE und R²-Score für das Test-Set
|
||||
test_mae = mean_absolute_error(test_labels, test_preds)
|
||||
test_rmse = np.sqrt(mean_squared_error(test_labels, test_preds))
|
||||
test_r2 = r2_score(test_labels, test_preds)
|
||||
print(f"Test RMSE: {test_rmse:.4f}, Test MAE: {test_mae:.4f}, Test R²: {test_r2:.4f}")
|
||||
|
||||
if test_rmse > best_params_rmse:
|
||||
best_params_rmse = test_rmse
|
||||
best_params = params
|
||||
|
||||
if len(grid_params) > 1:
|
||||
best_params_name = f'models/best_params_Transformer.json'
|
||||
with open(best_params_name, 'w') as f:
|
||||
json.dump(best_params, f)
|
||||
|
||||
if N_MODELS >1:
|
||||
# Ensemble Prediction
|
||||
|
|
|
|||
Loading…
Reference in New Issue