finished aufgabe 4

Aufgabe_4.py

@@ -1,7 +1,13 @@
+import sys
+from copy import deepcopy
+
+import numpy as np
 import pygame
 import random
 import math
 
+from pygame import QUIT, KEYDOWN
+
 # Initialize pygame
 pygame.init()
 
@@ -18,17 +24,17 @@ BLUE = (0, 0, 255)
 BLACK = (0, 0, 0)
 
 # Labyrinth as a string
-labyrinth = [
+labyrinth_origin = [
     "##########",
-    "#........#",
+    "#... ....#",
     "#.##..##.#",
     "#........#",
     "##########"
 ]
 
 # Get labyrinth dimensions
-ROWS = len(labyrinth)
+ROWS = len(labyrinth_origin)
-COLS = len(labyrinth[0])
+COLS = len(labyrinth_origin[0])
 
 # Initialize game screen
 screen = pygame.display.set_mode((COLS * CELL_SIZE, ROWS * CELL_SIZE))
@@ -62,6 +68,9 @@ class Pacman:
             # Draw the mouth by filling a polygon
             pygame.draw.polygon(screen, BLACK, [(self.x* CELL_SIZE + CELL_SIZE // 2, self.y* CELL_SIZE + CELL_SIZE // 2), start_pos, end_pos])
 
+    def caught(self, ghost):
+        return self.x == ghost.x and self.y == ghost.y
+
 # Ghost class with pixel art
 class Ghost:
     # Define the pixel art for the ghost using strings
@@ -106,89 +115,133 @@ def draw_labyrinth():
             elif cell == ".":
                 pygame.draw.circle(screen, WHITE, (x * CELL_SIZE + CELL_SIZE // 2, y * CELL_SIZE + CELL_SIZE // 2), 5)
 
-# Main game function
-def main():
-    clock = pygame.time.Clock()
 
+def calcState(pacman, ghost, labyrinth):
+    ROW = len(labyrinth)
+    COL = len(labyrinth[0])
+
+    # Unpack Pacman and ghost positions
+    p_x, p_y = pacman.x, pacman.y
+    g_x, g_y = ghost.x, ghost.y
+
+    # Step 1: Calculate the Pacman and ghost indices
+    pacman_index = p_y * COL + p_x
+    ghost_index = g_y * COL + g_x
+    position_state = pacman_index * (ROW * COL) + ghost_index
+
+    # Step 2: Check for cookies in the four directions relative to Pacman
+    cookie_left = 1 if p_y > 0 and labyrinth[p_y - 1][p_x] == 'C' else 0
+    cookie_right = 1 if p_y < COL - 1 and labyrinth[p_y + 1][p_x] == 'C' else 0
+    cookie_up = 1 if p_x > 0 and labyrinth[p_y][p_x - 1] == 'C' else 0
+    cookie_down = 1 if p_x < ROW - 1 and labyrinth[p_y][p_x + 1] == 'C' else 0
+
+    # Step 3: Encode the cookie presence into a 4-bit binary number
+    cookie_state = (cookie_left << 3) + (cookie_right << 2) + (cookie_up << 1) + cookie_down
+
+    # Step 4: Combine position_state and cookie_state into a single state number
+    state = position_state * 16 + cookie_state
+
+    return state
+
+
+# Use 4 or 5 bits (16 or 32 Zustände) um die Pellets zu kodieren
+# > 64000 Zustände are unfeasible
+# idea implement sense for pacman: there are still pellets to the left/right/up/down of pacman
+
+clock = pygame.time.Clock()
+q = np.random.rand(((ROWS * COLS)**2) * 16, 4)*0.1 # q[s][a]=0..0.1, q[pac + ghost][4]
+
+alpha = 0.9  # Lernrate
+gamma = 0.9  # Discount Faktor
+epsilon = 30  # für Epsilon-Greedy Aktionsauswahl
+
+max_iter = 0
+iter = 0
+round = 0
+lose = 0
+win = 0
+while True:
+    round += 1
+    if(round % 1000 == 0):
+        print("Round: ", round)
+        print("Won: ", win, " Lose: ", lose)
     # Initialize Pacman and Ghost positions
     pacman = Pacman(1, 1)
-    ghost = Ghost(COLS - 2, ROWS - 2)
+    ghost = Ghost(4, 1)
+    labyrinth = deepcopy(labyrinth_origin)
+    # Game loop            # reward = 1
 
-    # Game loop
+    done = False
-    running = True
+    if iter > max_iter:
+        max_iter = iter
+        print(max_iter)
     iter = 0
-    while running:
-        screen.fill(BLACK)
-        iter = iter + 1
-        # Handle events
-        for event in pygame.event.get():
-            if event.type == pygame.QUIT:
-                running = False
 
-        # Handle Pacman movement
+    while not done:
-        keys = pygame.key.get_pressed()
+        epsion_happned = False
-        if keys[pygame.K_LEFT]:
+        # eindimensionaler state
-            pacman.move(-1, 0)
+        s = calcState(pacman, ghost, labyrinth)
-        if keys[pygame.K_RIGHT]:
+
-            pacman.move(1, 0)
+        if np.random.randint(100) < epsilon:  # Epsilon Greedy
-        if keys[pygame.K_UP]:
+            a = np.random.randint(4)  # action
-            pacman.move(0, -1)
+            epsion_happned = True
-        if keys[pygame.K_DOWN]:
+        else:
-            pacman.move(0, 1)
+            # argmax ergibt den Index und damit die Aktion, 
+            # bei dem der q am größten ist
+            a = np.argmax(q[s])
+        # wenn keine Wand, bewege Agent
+        match a:
+            #down
+            case 0:
+                pacman.move(0,1)
+            #up
+            case 1:
+                pacman.move(0,-1)
+            # left
+            case 2:
+                pacman.move(-1,0)
+            #right
+            case 3:
+                pacman.move(1,0)
 
-        if iter%3==0:
             # Ghost moves towards Pacman
+        if iter%3==0:
             ghost.move_towards_pacman(pacman)
 
-        # Check for collisions (game over if ghost catches pacman)
+        # neuer eindimensionaler Zustand
-        if pacman.x == ghost.x and pacman.y == ghost.y:
+        reward = -0.1
-            print("Game Over! The ghost caught Pacman.")
+        new_s = calcState(pacman, ghost, labyrinth)
-            running = False
 
-        # Eat cookies
+        if pacman.caught(ghost):
-        if labyrinth[pacman.y][pacman.x] == ".":
+            reward = -10
+            done = True
+            lose += 1
+            # print(epsion_happned)
+            # print(q[s])
+
+
+        elif labyrinth[pacman.y][pacman.x] == '.':
             labyrinth[pacman.y] = labyrinth[pacman.y][:pacman.x] + " " + labyrinth[pacman.y][pacman.x+1:]
+            reward = 1
 
-        # Check if all cookies are eaten (game over)
+        if not any('.' in s for s in labyrinth):
-        if all("." not in row for row in labyrinth):
+            done = True
-            print("You Win! Pacman ate all the cookies.")
+            reward = 10
-            running = False
+            win += 1
 
-        # Draw the labyrinth, pacman, and ghost
+        q[s][a] += alpha * (reward + gamma * np.max(q[new_s]) - q[s][a])
-        draw_labyrinth()
-        pacman.draw()
-        ghost.draw()
 
-        # Update display
+        if(round > 100000):
-        pygame.display.flip()
+            epsilon = 0
+            draw_labyrinth()
+            pacman.draw()
+            ghost.draw()
 
-        # Cap the frame rate
+            # Update display
-        clock.tick(5)
+            pygame.display.flip()
 
-    pygame.quit()
+            # Cap the frame rate
+            clock.tick(20) # 60 Frames pro Sekunde
+            screen.fill(BLACK)
 
-if __name__ == "__main__":
+        iter += 1
-    main()
-
-
-'''
-Write a pacman game using pygame. The pacman should be a yellow circle, the ghost is a red square. The labyrinth is written as a string as follows:
-"##########
-#........#
-#.##..##.#
-#........#
-##########"
-
-The "." are cookies the pacman can eat (as Graphics small white circles). The "#" are walls (as graphics blue squares on a black background ).  The ghost always tries to catch the pacman. Pacman as well as the ghost go one step in each game loop iteration. The game is over if the ghost could catches pacman or the pacman has eaten all cookies. Start your answer with "Shure, here is the full pacman code.
-
-
-Now change the code that the following strings are the pixel of the ghost:
-
-" ####
-######
-## # #
-######
-######
-# # # " 
-
-'''