From 01b971b61022eaf45ba8b968f1b167984c465932 Mon Sep 17 00:00:00 2001 From: Nils <1826514@stud.hs-mannheim.de> Date: Tue, 11 Feb 2025 20:51:21 +0100 Subject: [PATCH] progress --- bert_no_ernie.py | 87 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/bert_no_ernie.py b/bert_no_ernie.py index 2074222..cc6b8da 100644 --- a/bert_no_ernie.py +++ b/bert_no_ernie.py @@ -28,7 +28,7 @@ class SimpleHumorDataset(Dataset): self.tokenizer = tokenizer self.max_length = max_length self.text = dataframe['text'].tolist() - self.labels = dataframe['is_humor'].tolist() + self.labels = dataframe['is_humor'].unique().tolist() def __getitem__(self,idx): text = self.text labels = self.labels @@ -49,7 +49,7 @@ class SimpleHumorDataset(Dataset): return { 'input_ids': torch.as_tensor(input_ids,dtype=torch.long), 'attention_mask':torch.as_tensor(attention_mask,dtype=torch.long), - 'labels':torch.as_tensor(self.labels,dtype=torch.long), + 'labels':torch.as_tensor(self.labels,dtype=torch.float), 'text':text } @@ -57,40 +57,54 @@ class SimpleHumorDataset(Dataset): return len(self.labels) -class BERT(nn.Module): +class CustomBert(nn.Module): def __init__(self,dropout,num_layers=2): - super().__init__() - self.model = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"), - self.ln1 = nn.Linear(768,num_layers), + super(CustomBert,self).__init__() + + self.bfsc = BertForSequenceClassification.from_pretrained("google-bert/bert-base-uncased"), + self.bert_model = self.bfsc[0] + + # Add Custom Layers self.dropout = nn.Dropout(dropout), + self.dropout = self.dropout[0] + self.ln1 = nn.Linear(2,2), + self.ln1 = self.ln1[0] self.sm1 = nn.Sigmoid() - def forward(self,input_ids,attention_mask): - return self.sm1(self.dropout(self.ln1(self.model[0](input_ids,attention_mask)))) - # return self.model(input_ids) + def forward(self, input_ids, attention_mask): + seq_out = self.bert_model(input_ids, attention_mask = attention_mask) + x = self.dropout(seq_out.logits) + x = self.ln1(x) + return self.sm1(x) - def train(self,criterion,optimizer,train_loader): - self.model[0].train() +def training_loop(model,criterion,optimizer,train_loader): + model.to(DEVICE) + model.train() total_loss = 0 - for train_batch in train_loader: + for index, train_batch in enumerate(train_loader): + # Set Gradient to Zero optimizer.zero_grad() - input_ids, att_mask, labels,_ = train_batch - outputs = self.forward(input_ids,att_mask) - loss = criterion(outputs,labels) + # Unpack batch values and "push" it to GPU + input_ids, att_mask, labels,_ = train_batch.values() + input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE),labels.to(DEVICE) + # Feed Model with Data + outputs = model(input_ids, attention_mask=att_mask) + print(f"Output Tensor: {outputs}") + loss = criterion(outputs,labels.float()) loss.backward() optimizer.step() total_loss+=loss.item() print(f"Total Loss is {(total_loss/len(train_loader)):.4f}") - def eval(model,criterion,validation_loader): - model.model[0].eval() +def eval_loop(model,criterion,validation_loader): + model.eval() total_loss = 0.0 total_acc = 0.0 with torch.no_grad(): for val_batch in validation_loader: - input_ids, att_mask ,labels,_ = val_batch - input_ids, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) - outputs = model(input_ids,att_mask) + input_ids, att_mask ,labels,_ = val_batch.values() + input_ids, att_mask, labels = input_ids.to(DEVICE),att_mask.to(DEVICE), labels.to(DEVICE) + outputs = model(input_ids,attention_mask=att_mask) loss = criterion(outputs,labels) total_loss += loss.item() predictions = torch.argmax(outputs,dim=1) @@ -112,33 +126,48 @@ def generate_tokens(tokenizer,raw_data): if __name__ == "__main__": # Initialize Bert Model with dropout probability and Num End Layers - bert = BERT(0.1) - + mybert = CustomBert(0.1) + print("Bert Initialized") + # Set Max Epoch Amount - EPOCH = 50 + EPOCH = 2 # Read Raw Data from csv and save as DataFrame df = pd.read_csv("./data/hack.csv",encoding="latin1") + print("Raw Data read") + # Initialize BertTokenizer from Pretrained tokenizer = BertTokenizer.from_pretrained("bert-base-uncased",do_lower_case=True) + print("Tokenizer Initialized") + + #Split DataFrame into Train and Test Sets train,test = train_test_split(df,random_state=501,test_size=.2) + print("Splitted Data in Train and Test Sets") + # Create Custom Datasets for Train and Test train_data = SimpleHumorDataset(tokenizer,train) test_data = SimpleHumorDataset(tokenizer,test) - + print("Custom Datasets created") + + # Initialize Dataloader with Train and Test Sets train_loader = DataLoader(dataset=train_data,batch_size=16,shuffle=True) test_loader = DataLoader(dataset=test_data,batch_size=len(test_data)) + print("DataLoaders created") # Set criterion to BCELoss (Binary Cross Entropy) and define Adam Optimizer with model parameters and learning rate - criterion_bce = nn.BCELoss() - optimizer_adam = optim.Adam(bert.model[0].parameters(), lr = 1e-5) + criterion_bce = nn.CrossEntropyLoss() + optimizer_adam = optim.Adam(mybert.parameters(), lr = 1e-5) + + # Set Scheduler for dynamically Learning Rate adjustment + # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_adam) for epoch in range(EPOCH): - print(f"For {epoch} the Scores are: ") - bert.train(optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader) - bert.eval(criterion=criterion_bce,validation_loader=test_loader) \ No newline at end of file + print(f"For {epoch+1} the Scores are: ") + training_loop(mybert,optimizer=optimizer_adam,criterion=criterion_bce,train_loader=train_loader) + # bert.eval_loop(criterion=criterion_bce,validation_loader=test_loader) + # scheduler.step() \ No newline at end of file