diff --git a/ReinforcmentLearning/game.py b/ReinforcmentLearning/game.py index f7cc00c..0eabfe3 100644 --- a/ReinforcmentLearning/game.py +++ b/ReinforcmentLearning/game.py @@ -2,10 +2,9 @@ import pygame import math import os -from ReinforcmentLearning.util import Direction, calc_current_state, epsilon_greedy, get_best_q_action +from ReinforcmentLearning.util import Direction, calc_current_state, epsilon_greedy, get_best_q_action, initial_q_fill # Initialize pygame -pygame.init() # Define constants SCREEN_WIDTH = 400 @@ -22,7 +21,7 @@ BLACK = (0, 0, 0) REWARD_ON_HALF = 50 # Labyrinth as a string -labyrinth_init = [ +LABYRINTH_INIT = [ "##########", "#........#", "#.##..##.#", @@ -30,25 +29,20 @@ labyrinth_init = [ "##########" ] -labyrinth = labyrinth_init.copy() - # Get labyrinth dimensions -ROWS = len(labyrinth) -COLS = len(labyrinth[0]) +ROWS = len(LABYRINTH_INIT) +COLS = len(LABYRINTH_INIT[0]) + -# Initialize game screen -screen = None -# screen = pygame.Surface((COLS * CELL_SIZE, ROWS * CELL_SIZE)) -# pygame.display.set_caption("Micro-Pacman") -# Pacman class class Pacman: - def __init__(self, x, y): + def __init__(self, screen, x, y): + self.screen = screen self.x = x self.y = y self.count = 0 - def move(self, dx, dy): + def move(self, labyrinth, dx, dy): new_x, new_y = self.x + dx, self.y + dy if labyrinth[new_y][new_x] != "#": self.x = new_x @@ -58,7 +52,7 @@ class Pacman: radius = CELL_SIZE // 2 - 4 start_angle = math.pi / 6 end_angle = -math.pi / 6 - pygame.draw.circle(screen, YELLOW, (self.x * CELL_SIZE + CELL_SIZE // 2, self.y * CELL_SIZE + CELL_SIZE // 2), CELL_SIZE // 2 - 4) + pygame.draw.circle(self.screen, YELLOW, (self.x * CELL_SIZE + CELL_SIZE // 2, self.y * CELL_SIZE + CELL_SIZE // 2), CELL_SIZE // 2 - 4) # Calculate the points for the mouth start_pos = (self.x* CELL_SIZE + CELL_SIZE // 2 + int(radius*1.3 * math.cos(start_angle)), self.y* CELL_SIZE + CELL_SIZE // 2 - int(radius*1.3 * math.sin(start_angle))) @@ -67,9 +61,9 @@ class Pacman: self.count += 1 if self.count%2==0: # Draw the mouth by filling a polygon - pygame.draw.polygon(screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos]) + pygame.draw.polygon(self.screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos]) + -# Ghost class with pixel art class Ghost: # Define the pixel art for the ghost using strings ghost_pixels = [ @@ -81,11 +75,12 @@ class Ghost: "# # # " ] - def __init__(self, x, y): + def __init__(self, screen, x, y): + self.screen = screen self.x = x self.y = y - def move_towards_pacman(self, pacman): + def move_towards_pacman(self, labyrinth, pacman): if self.x < pacman.x and labyrinth[self.y][self.x + 1] != "#": self.x += 1 elif self.x > pacman.x and labyrinth[self.y][self.x - 1] != "#": @@ -102,31 +97,42 @@ class Ghost: if pixel == "#": pixel_x = self.x * CELL_SIZE + col_idx * pixel_size pixel_y = self.y * CELL_SIZE + row_idx * pixel_size - pygame.draw.rect(screen, RED, (pixel_x, pixel_y, pixel_size, pixel_size)) + pygame.draw.rect(self.screen, RED, (pixel_x, pixel_y, pixel_size, pixel_size)) -# Draw walls and cookies -def draw_labyrinth(labyrinth): - for y, row in enumerate(labyrinth): - for x, cell in enumerate(row): - if cell == "#": - pygame.draw.rect(screen, BLUE, (x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE)) - elif cell == ".": - pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5) + + +def wrapper(AMOUNT_RUNS, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game): + #? Learning initial + q_values = {} + initial_q_fill(q_values) + + #? Game initial + pygame.init() + screen = None + + if show_game: + screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE)) + pygame.display.set_caption("Micro-Pacman") + + cookies_per_run = [] + for x in range(AMOUNT_RUNS): + amount_cookies_ate = run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game, screen) + cookies_per_run.append(amount_cookies_ate) + + pygame.quit() + + return cookies_per_run # Main game function -def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game): +def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game, screen): clock = pygame.time.Clock() - labyrinth = labyrinth_init.copy() - - if show_game: - pygame.display.set_caption("Micro-Pacman") - screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE)) + labyrinth = LABYRINTH_INIT.copy() # Initialize Pacman and Ghost positions - pacman = Pacman(1, 1) - ghost = Ghost(COLS - 2, ROWS - 2) + pacman = Pacman(screen, 1, 1) + ghost = Ghost(screen, COLS - 2, ROWS - 2) #? -------------------------MY CODE----------------------------------- state = calc_current_state(labyrinth, pacman.x, pacman.y, ghost.x, ghost.y) @@ -163,18 +169,18 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho #? -------------------------MY CODE----------------------------------- action = epsilon_greedy(q_values, state, EPSILON) if action == Direction.LEFT: - pacman.move(-1, 0) + pacman.move(labyrinth, -1, 0) if action == Direction.RIGHT: - pacman.move(1, 0) + pacman.move(labyrinth, 1, 0) if action == Direction.UP: - pacman.move(0, -1) + pacman.move(labyrinth, 0, -1) if action == Direction.DOWN: - pacman.move(0, 1) + pacman.move(labyrinth, 0, 1) #? -------------------------MY CODE----------------------------------- if iter%3==0: # Ghost moves towards Pacman - ghost.move_towards_pacman(pacman) + ghost.move_towards_pacman(labyrinth, pacman) # Check for collisions (game over if ghost catches pacman) if pacman.x == ghost.x and pacman.y == ghost.y: @@ -208,7 +214,6 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho reward = REWARD_ON_WIN running = False - # Draw the labyrinth, pacman, and ghost #? -------------------------MY CODE----------------------------------- if not running: @@ -234,8 +239,10 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho return 20-counter #? -------------------------MY CODE----------------------------------- + + # Draw the labyrinth, pacman, and ghost if show_game: - draw_labyrinth(labyrinth) + draw_labyrinth(screen, labyrinth) pacman.draw() ghost.draw() @@ -245,7 +252,16 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho # Cap the frame rate clock.tick(20) - pygame.quit() + +def draw_labyrinth(screen, labyrinth): + for y, row in enumerate(labyrinth): + for x, cell in enumerate(row): + if cell == "#": + pygame.draw.rect(screen, BLUE, (x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE)) + elif cell == ".": + pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5) + + if __name__ == "__main__": run_game() \ No newline at end of file diff --git a/ReinforcmentLearning/learning.py b/ReinforcmentLearning/learning.py index 26836c3..004a8ca 100644 --- a/ReinforcmentLearning/learning.py +++ b/ReinforcmentLearning/learning.py @@ -2,7 +2,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd -from ReinforcmentLearning.game import run_game +from ReinforcmentLearning.game import run_game, wrapper from ReinforcmentLearning.util import initial_q_fill @@ -27,16 +27,19 @@ def oneTry(EPSILON, ALPHA, GAMMA, AMOUNT_RUNS, REWARD_ON_WIN, REWARD_ON_LOSE, pl action: Direction q_value: (state, action) """ - - q_values = {} - initial_q_fill(q_values) + cookies_per_run = wrapper(AMOUNT_RUNS, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game) + + # if show_game: + # if x == AMOUNT_RUNS / 4: + # print("1 / 4 done") + + # if x == AMOUNT_RUNS / 2: + # print("2 / 4 done") + + # if x == (AMOUNT_RUNS / 2) + (AMOUNT_RUNS / 4): + # print("3 / 4 done") - cookies_per_run = [] - # Amount of single runs - for x in range(AMOUNT_RUNS): - amount_cookies_ate = run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game) - cookies_per_run.append(amount_cookies_ate) wins = sum(1 for result in cookies_per_run if result == 20) diff --git a/clean_game.py b/clean_game.py new file mode 100644 index 0000000..cdea1a7 --- /dev/null +++ b/clean_game.py @@ -0,0 +1,171 @@ +import pygame +import random +import math + +# Initialize pygame +pygame.init() + +# Define constants +SCREEN_WIDTH = 400 +SCREEN_HEIGHT = 400 +CELL_SIZE = 40 + +# Define colors +YELLOW = (255, 255, 0) +RED = (255, 0, 0) +WHITE = (255, 255, 255) +BLUE = (0, 0, 255) +BLACK = (0, 0, 0) + +# Labyrinth as a string +labyrinth = [ + "##########", + "#........#", + "#.##..##.#", + "#........#", + "##########" +] + +# Get labyrinth dimensions +ROWS = len(labyrinth) +COLS = len(labyrinth[0]) + +# Initialize game screen +screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE)) +pygame.display.set_caption("Micro-Pacman") + +# Pacman class +class Pacman: + def __init__(self, x, y): + self.x = x + self.y = y + self.count = 0 + + def move(self, dx, dy): + new_x, new_y = self.x + dx, self.y + dy + if labyrinth[new_y][new_x] != "#": + self.x = new_x + self.y = new_y + + def draw(self): + radius = CELL_SIZE // 2 - 4 + start_angle = math.pi / 6 + end_angle = -math.pi / 6 + pygame.draw.circle(screen, YELLOW, (self.x * CELL_SIZE + CELL_SIZE // 2, self.y * CELL_SIZE + CELL_SIZE // 2), CELL_SIZE // 2 - 4) + # Calculate the points for the mouth + start_pos = (self.x* CELL_SIZE + CELL_SIZE // 2 + int(radius*1.3 * math.cos(start_angle)), + self.y* CELL_SIZE + CELL_SIZE // 2 - int(radius*1.3 * math.sin(start_angle))) + end_pos = (self.x* CELL_SIZE + CELL_SIZE // 2 + int(radius*1.3 * math.cos(end_angle)), + self.y* CELL_SIZE + CELL_SIZE // 2 - int(radius*1.3 * math.sin(end_angle))) + self.count += 1 + if self.count%2==0: + # Draw the mouth by filling a polygon + pygame.draw.polygon(screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos]) + +# Ghost class with pixel art +class Ghost: + # Define the pixel art for the ghost using strings + ghost_pixels = [ + " #### ", + "######", + "## # #", + "######", + "######", + "# # # " + ] + + def __init__(self, x, y): + self.x = x + self.y = y + + def move_towards_pacman(self, pacman): + if self.x < pacman.x and labyrinth[self.y][self.x + 1] != "#": + self.x += 1 + elif self.x > pacman.x and labyrinth[self.y][self.x - 1] != "#": + self.x -= 1 + elif self.y < pacman.y and labyrinth[self.y + 1][self.x] != "#": + self.y += 1 + elif self.y > pacman.y and labyrinth[self.y - 1][self.x] != "#": + self.y -= 1 + + def draw(self): + pixel_size = CELL_SIZE // len(self.ghost_pixels) # Size of each pixel in the ghost art + for row_idx, row in enumerate(self.ghost_pixels): + for col_idx, pixel in enumerate(row): + if pixel == "#": + pixel_x = self.x * CELL_SIZE + col_idx * pixel_size + pixel_y = self.y * CELL_SIZE + row_idx * pixel_size + pygame.draw.rect(screen, RED, (pixel_x, pixel_y, pixel_size, pixel_size)) + +# Draw walls and cookies +def draw_labyrinth(): + for y, row in enumerate(labyrinth): + for x, cell in enumerate(row): + if cell == "#": + pygame.draw.rect(screen, BLUE, (x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE)) + elif cell == ".": + pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5) + +# Main game function +def main(): + clock = pygame.time.Clock() + + # Initialize Pacman and Ghost positions + pacman = Pacman(1, 1) + ghost = Ghost(COLS - 2, ROWS - 2) + + # Game loop + running = True + iter = 0 + while running: + screen.fill(BLACK) + iter = iter + 1 + # Handle events + for event in pygame.event.get(): + if event.type == pygame.QUIT: + running = False + + # Handle Pacman movement + keys = pygame.key.get_pressed() + if keys[pygame.K_LEFT]: + pacman.move(-1, 0) + if keys[pygame.K_RIGHT]: + pacman.move(1, 0) + if keys[pygame.K_UP]: + pacman.move(0, -1) + if keys[pygame.K_DOWN]: + pacman.move(0, 1) + + if iter%3==0: + # Ghost moves towards Pacman + ghost.move_towards_pacman(pacman) + + # Check for collisions (game over if ghost catches pacman) + if pacman.x == ghost.x and pacman.y == ghost.y: + print("Game Over! The ghost caught Pacman.") + running = False + + # Eat cookies + if labyrinth[pacman.y][pacman.x] == ".": + labyrinth[pacman.y] = labyrinth[pacman.y][:pacman.x] + " " + labyrinth[pacman.y][pacman.x+1:] + + # Check if all cookies are eaten (game over) + if all("." not in row for row in labyrinth): + print("You Win! Pacman ate all the cookies.") + running = False + + # Draw the labyrinth, pacman, and ghost + draw_labyrinth() + pacman.draw() + ghost.draw() + + # Update display + pygame.display.flip() + + # Cap the frame rate + clock.tick(5) + + pygame.quit() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/main.py b/main.py index e11684d..068283e 100644 --- a/main.py +++ b/main.py @@ -10,11 +10,11 @@ GAMMA = 0.2713 AMOUNT_RUNS = 5000 AMOUNT_TRIES = 5 -REWARD_ON_WIN = 100 -REWARD_ON_LOSE = -450 +REWARD_ON_WIN = 400 +REWARD_ON_LOSE = -250 plot_result = True -show_game = False +show_game = True oneTry(EPSILON, ALPHA, GAMMA, AMOUNT_RUNS, REWARD_ON_WIN, REWARD_ON_LOSE, plot_result, show_game)