dt

ReinforcmentLearning/game.py

@@ -2,10 +2,9 @@ import pygame
 import math
 import os
 
-from ReinforcmentLearning.util import Direction, calc_current_state, epsilon_greedy, get_best_q_action
+from ReinforcmentLearning.util import Direction, calc_current_state, epsilon_greedy, get_best_q_action, initial_q_fill
 
 # Initialize pygame
-pygame.init()
 
 # Define constants
 SCREEN_WIDTH = 400
@@ -22,7 +21,7 @@ BLACK = (0, 0, 0)
 REWARD_ON_HALF = 50
 
 # Labyrinth as a string
-labyrinth_init = [
+LABYRINTH_INIT = [
     "##########",
     "#........#",
     "#.##..##.#",
@@ -30,25 +29,20 @@ labyrinth_init = [
     "##########"
 ]
 
-labyrinth = labyrinth_init.copy()
-
 # Get labyrinth dimensions
-ROWS = len(labyrinth)
+ROWS = len(LABYRINTH_INIT)
-COLS = len(labyrinth[0])
+COLS = len(LABYRINTH_INIT[0])
+
 
-# Initialize game screen
-screen = None
-# screen = pygame.Surface((COLS * CELL_SIZE, ROWS * CELL_SIZE))
-# pygame.display.set_caption("Micro-Pacman")
 
-# Pacman class
 class Pacman:
-    def __init__(self, x, y):
+    def __init__(self, screen, x, y):
+        self.screen = screen
         self.x = x
         self.y = y
         self.count = 0
 
-    def move(self, dx, dy):
+    def move(self, labyrinth, dx, dy):
         new_x, new_y = self.x + dx, self.y + dy
         if labyrinth[new_y][new_x] != "#":
             self.x = new_x
@@ -58,7 +52,7 @@ class Pacman:
         radius = CELL_SIZE // 2 - 4
         start_angle = math.pi / 6
         end_angle = -math.pi / 6
-        pygame.draw.circle(screen, YELLOW, (self.x * CELL_SIZE + CELL_SIZE // 2, self.y * CELL_SIZE + CELL_SIZE // 2), CELL_SIZE // 2 - 4)
+        pygame.draw.circle(self.screen, YELLOW, (self.x * CELL_SIZE + CELL_SIZE // 2, self.y * CELL_SIZE + CELL_SIZE // 2), CELL_SIZE // 2 - 4)
             # Calculate the points for the mouth
         start_pos = (self.x* CELL_SIZE + CELL_SIZE // 2 + int(radius*1.3 * math.cos(start_angle)),
                      self.y* CELL_SIZE + CELL_SIZE // 2 - int(radius*1.3 * math.sin(start_angle)))
@@ -67,9 +61,9 @@ class Pacman:
         self.count += 1
         if self.count%2==0:
             # Draw the mouth by filling a polygon
-            pygame.draw.polygon(screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos])
+            pygame.draw.polygon(self.screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos])
+
 
-# Ghost class with pixel art
 class Ghost:
     # Define the pixel art for the ghost using strings
     ghost_pixels = [
@@ -81,11 +75,12 @@ class Ghost:
         "# # # "
     ]
 
-    def __init__(self, x, y):
+    def __init__(self, screen, x, y):
+        self.screen = screen
         self.x = x
         self.y = y
 
-    def move_towards_pacman(self, pacman):
+    def move_towards_pacman(self, labyrinth, pacman):
         if self.x < pacman.x and labyrinth[self.y][self.x + 1] != "#":
             self.x += 1
         elif self.x > pacman.x and labyrinth[self.y][self.x - 1] != "#":
@@ -102,31 +97,42 @@ class Ghost:
                 if pixel == "#":
                     pixel_x = self.x * CELL_SIZE + col_idx * pixel_size
                     pixel_y = self.y * CELL_SIZE + row_idx * pixel_size
-                    pygame.draw.rect(screen, RED, (pixel_x, pixel_y, pixel_size, pixel_size))
+                    pygame.draw.rect(self.screen, RED, (pixel_x, pixel_y, pixel_size, pixel_size))
 
-# Draw walls and cookies
+
-def draw_labyrinth(labyrinth):
+
-    for y, row in enumerate(labyrinth):
+def wrapper(AMOUNT_RUNS, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game):
-        for x, cell in enumerate(row):
+    #? Learning initial
-            if cell == "#":
+    q_values = {}
-                pygame.draw.rect(screen, BLUE, (x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE))
+    initial_q_fill(q_values)
-            elif cell == ".":
+    
-                pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5)
+    #? Game initial
+    pygame.init()
+    screen = None
+
+    if show_game:
+        screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE))
+        pygame.display.set_caption("Micro-Pacman")
+
+    cookies_per_run = []
+    for x in range(AMOUNT_RUNS):
+        amount_cookies_ate = run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game, screen)
+        cookies_per_run.append(amount_cookies_ate)
+
+    pygame.quit()
+
+    return cookies_per_run
 
 
 # Main game function
-def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game):
+def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game, screen):
     clock = pygame.time.Clock()
-    labyrinth = labyrinth_init.copy()
+    labyrinth = LABYRINTH_INIT.copy()
-
-    if show_game:
-        pygame.display.set_caption("Micro-Pacman")
-        screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE))
 
 
     # Initialize Pacman and Ghost positions
-    pacman = Pacman(1, 1)
+    pacman = Pacman(screen, 1, 1)
-    ghost = Ghost(COLS - 2, ROWS - 2)
+    ghost = Ghost(screen, COLS - 2, ROWS - 2)
 
     #? -------------------------MY CODE-----------------------------------
     state = calc_current_state(labyrinth, pacman.x, pacman.y, ghost.x, ghost.y)
@@ -163,18 +169,18 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho
         #? -------------------------MY CODE-----------------------------------
         action = epsilon_greedy(q_values, state, EPSILON)
         if action == Direction.LEFT:
-            pacman.move(-1, 0)
+            pacman.move(labyrinth, -1, 0)
         if action == Direction.RIGHT:
-            pacman.move(1, 0)
+            pacman.move(labyrinth, 1, 0)
         if action == Direction.UP:
-            pacman.move(0, -1)
+            pacman.move(labyrinth, 0, -1)
         if action == Direction.DOWN:
-            pacman.move(0, 1)
+            pacman.move(labyrinth, 0, 1)
         #? -------------------------MY CODE-----------------------------------
 
         if iter%3==0:
             # Ghost moves towards Pacman
-            ghost.move_towards_pacman(pacman)
+            ghost.move_towards_pacman(labyrinth, pacman)
 
         # Check for collisions (game over if ghost catches pacman)
         if pacman.x == ghost.x and pacman.y == ghost.y:
@@ -208,7 +214,6 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho
             reward = REWARD_ON_WIN
             running = False
 
-        # Draw the labyrinth, pacman, and ghost
 
         #? -------------------------MY CODE-----------------------------------
         if not running:
@@ -234,8 +239,10 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho
             return 20-counter
         #? -------------------------MY CODE-----------------------------------
         
+
+        # Draw the labyrinth, pacman, and ghost
         if show_game:
-            draw_labyrinth(labyrinth)
+            draw_labyrinth(screen, labyrinth)
             pacman.draw()
             ghost.draw()
 
@@ -245,7 +252,16 @@ def run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, sho
             # Cap the frame rate
             clock.tick(20)
 
-    pygame.quit()
+
+def draw_labyrinth(screen, labyrinth):
+    for y, row in enumerate(labyrinth):
+        for x, cell in enumerate(row):
+            if cell == "#":
+                pygame.draw.rect(screen, BLUE, (x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE))
+            elif cell == ".":
+                pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5)
+
+
 
 if __name__ == "__main__":
     run_game()

ReinforcmentLearning/learning.py

@@ -2,7 +2,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 
-from ReinforcmentLearning.game import run_game
+from ReinforcmentLearning.game import run_game, wrapper
 from ReinforcmentLearning.util import initial_q_fill
 
 
@@ -27,16 +27,19 @@ def oneTry(EPSILON, ALPHA, GAMMA, AMOUNT_RUNS, REWARD_ON_WIN, REWARD_ON_LOSE, pl
     action: Direction
     q_value: (state, action)
     """
-    
-    q_values = {}
 
-    initial_q_fill(q_values)
+    cookies_per_run = wrapper(AMOUNT_RUNS, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game)
+
+        # if show_game:
+        #     if x == AMOUNT_RUNS / 4:
+        #         print("1 / 4 done")
+
+        #     if x == AMOUNT_RUNS / 2:
+        #         print("2 / 4 done")
+
+        #     if x == (AMOUNT_RUNS / 2) + (AMOUNT_RUNS / 4):
+        #         print("3 / 4 done")
 
-    cookies_per_run = []
-    # Amount of single runs
-    for x in range(AMOUNT_RUNS):
-        amount_cookies_ate = run_game(q_values, EPSILON, ALPHA, GAMMA, REWARD_ON_WIN, REWARD_ON_LOSE, show_game)
-        cookies_per_run.append(amount_cookies_ate)
 
     wins = sum(1 for result in cookies_per_run if result == 20)
 

clean_game.py

@@ -0,0 +1,171 @@
+import pygame
+import random
+import math
+
+# Initialize pygame
+pygame.init()
+
+# Define constants
+SCREEN_WIDTH = 400
+SCREEN_HEIGHT = 400
+CELL_SIZE = 40
+
+# Define colors
+YELLOW = (255, 255, 0)
+RED = (255, 0, 0)
+WHITE = (255, 255, 255)
+BLUE = (0, 0, 255)
+BLACK = (0, 0, 0)
+
+# Labyrinth as a string
+labyrinth = [
+    "##########",
+    "#........#",
+    "#.##..##.#",
+    "#........#",
+    "##########"
+]
+
+# Get labyrinth dimensions
+ROWS = len(labyrinth)
+COLS = len(labyrinth[0])
+
+# Initialize game screen
+screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE))
+pygame.display.set_caption("Micro-Pacman")
+
+# Pacman class
+class Pacman:
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+        self.count = 0
+
+    def move(self, dx, dy):
+        new_x, new_y = self.x + dx, self.y + dy
+        if labyrinth[new_y][new_x] != "#":
+            self.x = new_x
+            self.y = new_y
+
+    def draw(self):
+        radius = CELL_SIZE // 2 - 4
+        start_angle = math.pi / 6
+        end_angle = -math.pi / 6
+        pygame.draw.circle(screen, YELLOW, (self.x * CELL_SIZE + CELL_SIZE // 2, self.y * CELL_SIZE + CELL_SIZE // 2), CELL_SIZE // 2 - 4)
+            # Calculate the points for the mouth
+        start_pos = (self.x* CELL_SIZE + CELL_SIZE // 2 + int(radius*1.3 * math.cos(start_angle)),
+                     self.y* CELL_SIZE + CELL_SIZE // 2 - int(radius*1.3 * math.sin(start_angle)))
+        end_pos = (self.x* CELL_SIZE + CELL_SIZE // 2 + int(radius*1.3 * math.cos(end_angle)),
+                   self.y* CELL_SIZE + CELL_SIZE // 2 - int(radius*1.3 * math.sin(end_angle)))
+        self.count += 1
+        if self.count%2==0:
+            # Draw the mouth by filling a polygon
+            pygame.draw.polygon(screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos])
+
+# Ghost class with pixel art
+class Ghost:
+    # Define the pixel art for the ghost using strings
+    ghost_pixels = [
+        " #### ",
+        "######",
+        "## # #",
+        "######",
+        "######",
+        "# # # "
+    ]
+
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+    def move_towards_pacman(self, pacman):
+        if self.x < pacman.x and labyrinth[self.y][self.x + 1] != "#":
+            self.x += 1
+        elif self.x > pacman.x and labyrinth[self.y][self.x - 1] != "#":
+            self.x -= 1
+        elif self.y < pacman.y and labyrinth[self.y + 1][self.x] != "#":
+            self.y += 1
+        elif self.y > pacman.y and labyrinth[self.y - 1][self.x] != "#":
+            self.y -= 1
+
+    def draw(self):
+        pixel_size = CELL_SIZE // len(self.ghost_pixels)  # Size of each pixel in the ghost art
+        for row_idx, row in enumerate(self.ghost_pixels):
+            for col_idx, pixel in enumerate(row):
+                if pixel == "#":
+                    pixel_x = self.x * CELL_SIZE + col_idx * pixel_size
+                    pixel_y = self.y * CELL_SIZE + row_idx * pixel_size
+                    pygame.draw.rect(screen, RED, (pixel_x, pixel_y, pixel_size, pixel_size))
+
+# Draw walls and cookies
+def draw_labyrinth():
+    for y, row in enumerate(labyrinth):
+        for x, cell in enumerate(row):
+            if cell == "#":
+                pygame.draw.rect(screen, BLUE, (x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE))
+            elif cell == ".":
+                pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5)
+
+# Main game function
+def main():
+    clock = pygame.time.Clock()
+
+    # Initialize Pacman and Ghost positions
+    pacman = Pacman(1, 1)
+    ghost = Ghost(COLS - 2, ROWS - 2)
+
+    # Game loop
+    running = True
+    iter = 0
+    while running:
+        screen.fill(BLACK)
+        iter = iter + 1
+        # Handle events
+        for event in pygame.event.get():
+            if event.type == pygame.QUIT:
+                running = False
+
+        # Handle Pacman movement
+        keys = pygame.key.get_pressed()
+        if keys[pygame.K_LEFT]:
+            pacman.move(-1, 0)
+        if keys[pygame.K_RIGHT]:
+            pacman.move(1, 0)
+        if keys[pygame.K_UP]:
+            pacman.move(0, -1)
+        if keys[pygame.K_DOWN]:
+            pacman.move(0, 1)
+
+        if iter%3==0:
+            # Ghost moves towards Pacman
+            ghost.move_towards_pacman(pacman)
+
+        # Check for collisions (game over if ghost catches pacman)
+        if pacman.x == ghost.x and pacman.y == ghost.y:
+            print("Game Over! The ghost caught Pacman.")
+            running = False
+
+        # Eat cookies
+        if labyrinth[pacman.y][pacman.x] == ".":
+            labyrinth[pacman.y] = labyrinth[pacman.y][:pacman.x] + " " + labyrinth[pacman.y][pacman.x+1:]
+
+        # Check if all cookies are eaten (game over)
+        if all("." not in row for row in labyrinth):
+            print("You Win! Pacman ate all the cookies.")
+            running = False
+
+        # Draw the labyrinth, pacman, and ghost
+        draw_labyrinth()
+        pacman.draw()
+        ghost.draw()
+
+        # Update display
+        pygame.display.flip()
+
+        # Cap the frame rate
+        clock.tick(5)
+
+    pygame.quit()
+
+if __name__ == "__main__":
+    main()

main.py

@@ -10,11 +10,11 @@ GAMMA = 0.2713
 AMOUNT_RUNS = 5000
 AMOUNT_TRIES = 5
 
-REWARD_ON_WIN = 100
+REWARD_ON_WIN = 400
-REWARD_ON_LOSE = -450
+REWARD_ON_LOSE = -250
 
 plot_result = True
-show_game = False
+show_game = True
 
 
 oneTry(EPSILON, ALPHA, GAMMA, AMOUNT_RUNS, REWARD_ON_WIN, REWARD_ON_LOSE, plot_result, show_game)