Major code restructuring: Reorganized project structure, added AI components - Added new module structure (ai, config, core, ui) - Moved snake and food logic into core module - Added training configuration - Updated gitignore for project-specific files - Modified tests to match new structure
This commit is contained in:
1 parent
251822ec35
commit
b8eeaa4739
34 files changed
+3928
-513
No files matched your search
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
Training Configuration UI
|
||||
|
||||
This module provides a graphical interface for configuring AI training parameters.
|
||||
"""
|
||||
|
||||
import pygame
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, Optional, Tuple, List
|
||||
|
||||
class ConfigUI:
|
||||
def __init__(self, width: int = 800, height: int = 600):
|
||||
"""Initialize the configuration UI."""
|
||||
pygame.init()
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.screen = pygame.display.set_mode((width, height))
|
||||
pygame.display.set_caption("Training Configuration")
|
||||
|
||||
# Fonts
|
||||
self.title_font = pygame.font.Font(None, 48)
|
||||
self.header_font = pygame.font.Font(None, 36)
|
||||
self.text_font = pygame.font.Font(None, 24)
|
||||
|
||||
# Colors
|
||||
self.colors = {
|
||||
'background': (0, 0, 0),
|
||||
'text': (200, 200, 200),
|
||||
'highlight': (0, 255, 0),
|
||||
'button': (40, 40, 40),
|
||||
'button_hover': (60, 60, 60),
|
||||
'input_bg': (30, 30, 30),
|
||||
'input_active': (50, 50, 50)
|
||||
}
|
||||
|
||||
# Parameters
|
||||
self.parameters = {
|
||||
'timesteps': {
|
||||
'value': 1000000,
|
||||
'type': 'int',
|
||||
'min': 1000,
|
||||
'max': 10000000,
|
||||
'description': 'Total timesteps to train for'
|
||||
},
|
||||
'learning_rate': {
|
||||
'value': 0.0003,
|
||||
'type': 'float',
|
||||
'min': 0.00001,
|
||||
'max': 0.01,
|
||||
'description': 'Learning rate for training'
|
||||
},
|
||||
'batch_size': {
|
||||
'value': 64,
|
||||
'type': 'int',
|
||||
'min': 32,
|
||||
'max': 512,
|
||||
'description': 'Batch size for training'
|
||||
},
|
||||
'n_envs': {
|
||||
'value': 8,
|
||||
'type': 'int',
|
||||
'min': 1,
|
||||
'max': 32,
|
||||
'description': 'Number of parallel environments'
|
||||
},
|
||||
'n_steps': {
|
||||
'value': 2048,
|
||||
'type': 'int',
|
||||
'min': 128,
|
||||
'max': 8192,
|
||||
'description': 'Number of steps per update'
|
||||
}
|
||||
}
|
||||
|
||||
# UI state
|
||||
self.active_input = None
|
||||
self.input_text = ""
|
||||
self.scroll_offset = 0
|
||||
self.max_scroll = max(0, len(self.parameters) * 60 - (height - 200))
|
||||
|
||||
# Load saved config if exists
|
||||
self.config_file = "training_config.json"
|
||||
self.load_config()
|
||||
|
||||
def load_config(self) -> None:
|
||||
"""Load configuration from file."""
|
||||
if os.path.exists(self.config_file):
|
||||
try:
|
||||
with open(self.config_file, 'r') as f:
|
||||
saved_config = json.load(f)
|
||||
for key, value in saved_config.items():
|
||||
if key in self.parameters:
|
||||
self.parameters[key]['value'] = value
|
||||
except:
|
||||
pass
|
||||
|
||||
def save_config(self) -> None:
|
||||
"""Save configuration to file."""
|
||||
config = {key: param['value'] for key, param in self.parameters.items()}
|
||||
with open(self.config_file, 'w') as f:
|
||||
json.dump(config, f, indent=4)
|
||||
|
||||
def draw_text_input(self, rect: pygame.Rect, value: Any, active: bool) -> None:
|
||||
"""Draw a text input field."""
|
||||
color = self.colors['input_active'] if active else self.colors['input_bg']
|
||||
pygame.draw.rect(self.screen, color, rect)
|
||||
pygame.draw.rect(self.screen, self.colors['text'], rect, 1)
|
||||
|
||||
text = str(value)
|
||||
if active:
|
||||
text = self.input_text + "|"
|
||||
|
||||
text_surface = self.text_font.render(text, True, self.colors['text'])
|
||||
text_rect = text_surface.get_rect(midleft=(rect.left + 5, rect.centery))
|
||||
self.screen.blit(text_surface, text_rect)
|
||||
|
||||
def draw_button(self, rect: pygame.Rect, text: str, hover: bool = False) -> None:
|
||||
"""Draw a button."""
|
||||
color = self.colors['button_hover'] if hover else self.colors['button']
|
||||
pygame.draw.rect(self.screen, color, rect)
|
||||
pygame.draw.rect(self.screen, self.colors['text'], rect, 1)
|
||||
|
||||
text_surface = self.text_font.render(text, True, self.colors['text'])
|
||||
text_rect = text_surface.get_rect(center=rect.center)
|
||||
self.screen.blit(text_surface, text_rect)
|
||||
|
||||
def validate_input(self, param: Dict[str, Any], value: str) -> Optional[Any]:
|
||||
"""Validate and convert input value."""
|
||||
try:
|
||||
if param['type'] == 'int':
|
||||
val = int(value)
|
||||
else:
|
||||
val = float(value)
|
||||
|
||||
if val < param['min'] or val > param['max']:
|
||||
return None
|
||||
return val
|
||||
except:
|
||||
return None
|
||||
|
||||
def run(self) -> Optional[Dict[str, Any]]:
|
||||
"""Run the configuration UI. Returns the config dict if saved, None if cancelled."""
|
||||
running = True
|
||||
save_clicked = False
|
||||
mouse_pos = (0, 0)
|
||||
|
||||
while running:
|
||||
for event in pygame.event.get():
|
||||
if event.type == pygame.QUIT:
|
||||
running = False
|
||||
|
||||
elif event.type == pygame.MOUSEBUTTONDOWN:
|
||||
mouse_pos = event.pos
|
||||
|
||||
# Check parameter inputs
|
||||
y = 100 - self.scroll_offset
|
||||
for name, param in self.parameters.items():
|
||||
input_rect = pygame.Rect(300, y, 200, 30)
|
||||
if input_rect.collidepoint(mouse_pos):
|
||||
self.active_input = name
|
||||
self.input_text = str(param['value'])
|
||||
y += 60
|
||||
|
||||
# Check buttons
|
||||
save_rect = pygame.Rect(self.width//2 - 150, self.height - 60, 140, 40)
|
||||
cancel_rect = pygame.Rect(self.width//2 + 10, self.height - 60, 140, 40)
|
||||
|
||||
if save_rect.collidepoint(mouse_pos):
|
||||
self.save_config()
|
||||
save_clicked = True
|
||||
running = False
|
||||
elif cancel_rect.collidepoint(mouse_pos):
|
||||
running = False
|
||||
|
||||
elif event.type == pygame.MOUSEBUTTONUP:
|
||||
if event.button == 4: # Mouse wheel up
|
||||
self.scroll_offset = max(0, self.scroll_offset - 30)
|
||||
elif event.button == 5: # Mouse wheel down
|
||||
self.scroll_offset = min(self.max_scroll, self.scroll_offset + 30)
|
||||
|
||||
elif event.type == pygame.KEYDOWN:
|
||||
if self.active_input is not None:
|
||||
if event.key == pygame.K_RETURN:
|
||||
param = self.parameters[self.active_input]
|
||||
if val := self.validate_input(param, self.input_text):
|
||||
param['value'] = val
|
||||
self.active_input = None
|
||||
elif event.key == pygame.K_BACKSPACE:
|
||||
self.input_text = self.input_text[:-1]
|
||||
else:
|
||||
if event.unicode.isnumeric() or event.unicode == '.':
|
||||
self.input_text += event.unicode
|
||||
|
||||
# Draw UI
|
||||
self.screen.fill(self.colors['background'])
|
||||
|
||||
# Title
|
||||
title = "Training Configuration"
|
||||
title_surface = self.title_font.render(title, True, self.colors['highlight'])
|
||||
title_rect = title_surface.get_rect(midtop=(self.width//2, 20))
|
||||
self.screen.blit(title_surface, title_rect)
|
||||
|
||||
# Parameters
|
||||
y = 100 - self.scroll_offset
|
||||
for name, param in self.parameters.items():
|
||||
if 0 <= y <= self.height - 100:
|
||||
# Parameter name and description
|
||||
name_surface = self.header_font.render(name, True, self.colors['text'])
|
||||
desc_surface = self.text_font.render(param['description'], True, self.colors['text'])
|
||||
self.screen.blit(name_surface, (20, y))
|
||||
self.screen.blit(desc_surface, (20, y + 30))
|
||||
|
||||
# Input field
|
||||
input_rect = pygame.Rect(300, y, 200, 30)
|
||||
self.draw_text_input(input_rect, param['value'], name == self.active_input)
|
||||
y += 60
|
||||
|
||||
# Buttons
|
||||
save_rect = pygame.Rect(self.width//2 - 150, self.height - 60, 140, 40)
|
||||
cancel_rect = pygame.Rect(self.width//2 + 10, self.height - 60, 140, 40)
|
||||
|
||||
save_hover = save_rect.collidepoint(mouse_pos)
|
||||
cancel_hover = cancel_rect.collidepoint(mouse_pos)
|
||||
|
||||
self.draw_button(save_rect, "Save", save_hover)
|
||||
self.draw_button(cancel_rect, "Cancel", cancel_hover)
|
||||
|
||||
pygame.display.flip()
|
||||
|
||||
pygame.quit()
|
||||
|
||||
if save_clicked:
|
||||
return {key: param['value'] for key, param in self.parameters.items()}
|
||||
return None
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
AI Controller for Snake Game
|
||||
|
||||
This module provides the AI controller that uses trained models to play the game.
|
||||
It handles model loading, state processing, and decision making during gameplay.
|
||||
"""
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
from stable_baselines3 import PPO
|
||||
from ai.environment import SnakeEnv, Direction
|
||||
|
||||
class AIController:
|
||||
"""AI controller that uses trained models to play the game."""
|
||||
|
||||
def __init__(self, difficulty: str = "medium"):
|
||||
"""
|
||||
Initialize the AI controller.
|
||||
|
||||
Args:
|
||||
difficulty: "easy", "medium", or "hard"
|
||||
"""
|
||||
self.difficulty = difficulty
|
||||
self.model = None
|
||||
self.env = SnakeEnv() # For state processing
|
||||
self._load_model()
|
||||
|
||||
def _load_model(self):
|
||||
"""Load the appropriate model based on difficulty."""
|
||||
model_path = f"models/{self.difficulty}/best_model.zip"
|
||||
if not os.path.exists(model_path):
|
||||
# Fall back to final model if best model doesn't exist
|
||||
model_path = f"models/{self.difficulty}/final_model.zip"
|
||||
|
||||
if not os.path.exists(model_path):
|
||||
raise FileNotFoundError(
|
||||
f"No model found for difficulty {self.difficulty}. "
|
||||
"Please train the model first."
|
||||
)
|
||||
|
||||
self.model = PPO.load(model_path)
|
||||
|
||||
def get_action(self, game_state: dict) -> Direction:
|
||||
"""
|
||||
Get the next action based on the current game state.
|
||||
|
||||
Args:
|
||||
game_state: Dictionary containing:
|
||||
- snake: Snake object
|
||||
- food: Food object
|
||||
- width: Game width
|
||||
- height: Game height
|
||||
|
||||
Returns:
|
||||
Direction enum indicating the chosen action
|
||||
"""
|
||||
# Update environment with current game state
|
||||
self.env.snake = game_state["snake"]
|
||||
self.env.food = game_state["food"]
|
||||
self.env.width = game_state["width"]
|
||||
self.env.height = game_state["height"]
|
||||
|
||||
# Get state observation
|
||||
state = self.env._get_state()
|
||||
|
||||
# Get action from model
|
||||
action, _ = self.model.predict(state, deterministic=True)
|
||||
|
||||
# Convert action index to Direction
|
||||
return self.env.action_space[action]
|
||||
@@ -0,0 +1,787 @@
|
||||
"""
|
||||
Snake Game Environment for Reinforcement Learning
|
||||
|
||||
This module provides a gym-like interface for training AI agents to play the snake game.
|
||||
It includes:
|
||||
- State observation space
|
||||
- Action space
|
||||
- Reward system
|
||||
- Environment dynamics
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import random
|
||||
import gymnasium as gym
|
||||
from gymnasium import spaces
|
||||
from typing import Tuple, List, Dict, Any, Optional
|
||||
import pygame
|
||||
from src.core import Snake
|
||||
from src.core import Direction
|
||||
from src.core import GameSession
|
||||
from src.ui import GameArea
|
||||
from src.config import GameRules
|
||||
|
||||
def manhattan_distance(pos1: Tuple[int, int], pos2: Tuple[int, int]) -> int:
|
||||
"""Calculate Manhattan distance between two points."""
|
||||
return abs(pos1[0] - pos2[0]) + abs(pos1[1] - pos2[1])
|
||||
|
||||
def manhattan_distance_wrap(pos1: Tuple[int, int], pos2: Tuple[int, int], size: int) -> int:
|
||||
"""Calculate Manhattan distance between two points with wrap-around."""
|
||||
dx = abs(pos1[0] - pos2[0])
|
||||
dy = abs(pos1[1] - pos2[1])
|
||||
return min(dx, size - dx) + min(dy, size - dy)
|
||||
|
||||
class SnakeEnv(gym.Env):
|
||||
"""A Gymnasium environment for training snake AI agents."""
|
||||
|
||||
metadata = {"render_modes": ["human", "rgb_array"], "render_fps": 30}
|
||||
|
||||
def __init__(self, size=30, render_mode=None, difficulty="easy"):
|
||||
super().__init__()
|
||||
self.size = size
|
||||
self.render_mode = render_mode
|
||||
self.difficulty = difficulty
|
||||
|
||||
# Action to Direction mapping
|
||||
self.action_to_direction = {
|
||||
0: Direction.UP,
|
||||
1: Direction.RIGHT,
|
||||
2: Direction.DOWN,
|
||||
3: Direction.LEFT
|
||||
}
|
||||
|
||||
# Direction to index mapping for one-hot encoding
|
||||
self.direction_to_index = {
|
||||
Direction.UP: 0,
|
||||
Direction.RIGHT: 1,
|
||||
Direction.DOWN: 2,
|
||||
Direction.LEFT: 3
|
||||
}
|
||||
|
||||
# Initialize reward scales with emphasis on precision
|
||||
self.reward_scales = {
|
||||
'food': 20.0, # Base food reward (will be scaled exponentially)
|
||||
'death': -5.0, # Reduced death penalty to encourage exploration
|
||||
'distance': 0.5, # Increased distance reward
|
||||
'survival': -0.01, # Small survival penalty
|
||||
'milestone': 10.0, # Larger milestone rewards
|
||||
'efficiency': 0.2, # Small efficiency reward
|
||||
'exploration': 0.5, # Increased exploration reward
|
||||
'safety': 1.0, # Safety reward for avoiding danger
|
||||
'timeout': -1.0, # Reduced timeout penalty
|
||||
'wrap_bonus': 0.2, # Small wrap-around bonus
|
||||
'near_miss': -0.1, # Very small near miss penalty
|
||||
'repetitive': -2 # Increased base penalty for repetitive movement
|
||||
}
|
||||
|
||||
# Near miss detection
|
||||
self.near_miss_threshold = 1 # Only penalize very close misses
|
||||
|
||||
# Time limit parameters - more lenient
|
||||
self.base_time_limit = 200 # Double base time limit
|
||||
self.length_time_bonus = 100 # Double length bonus
|
||||
self.max_time_limit = 1000 # Much longer maximum time
|
||||
|
||||
# Exploration decay
|
||||
self.initial_exploration_bonus = 2.0 # Higher initial exploration
|
||||
self.exploration_decay = 0.9995 # Slower decay
|
||||
self.min_exploration_bonus = 0.5 # Higher minimum exploration
|
||||
self.current_exploration_bonus = self.initial_exploration_bonus
|
||||
self.total_episodes = 0 # Track number of episodes for decay
|
||||
|
||||
# Curriculum learning parameters
|
||||
self.curriculum_stage = 0
|
||||
self.success_threshold = {
|
||||
0: 3, # Stage 0: Basic movement (need more consistent success)
|
||||
1: 5, # Stage 1: Food collection
|
||||
2: 7, # Stage 2: Longer snake
|
||||
3: 10 # Stage 3: Full difficulty
|
||||
}
|
||||
self.consecutive_successes = 0
|
||||
self.stage_requirements = {
|
||||
0: 1, # Stage 0: Get one food
|
||||
1: 3, # Stage 1: Get three food
|
||||
2: 5, # Stage 2: Get five food
|
||||
3: 7 # Stage 3: Get seven food
|
||||
}
|
||||
|
||||
# Dynamic episode length based on curriculum stage
|
||||
self.base_max_steps = 300 # More steps for exploration
|
||||
self.max_steps = self.base_max_steps
|
||||
|
||||
# Window dimensions for rendering
|
||||
self.window_width = 1024
|
||||
self.window_height = 768
|
||||
|
||||
# Create game area and rules
|
||||
self.game_area = GameArea(self.window_width, self.window_height)
|
||||
self.rules = GameRules()
|
||||
self.rules.wrap_around = False # No wrap-around for AI training
|
||||
|
||||
# Define action and observation spaces
|
||||
self.action_space = spaces.Discrete(4) # Up, Right, Down, Left
|
||||
|
||||
num_channels = 26 # Total number of channels we created in _get_normalized_observation
|
||||
self.observation_space = spaces.Box(
|
||||
low=0,
|
||||
high=1,
|
||||
shape=(num_channels, size, size), # (channels, height, width)
|
||||
dtype=np.float32
|
||||
)
|
||||
|
||||
# Game session
|
||||
self.session = None
|
||||
self.steps = 0
|
||||
self.steps_since_food = 0
|
||||
self.current_time = 0
|
||||
self.last_score = 0
|
||||
self.last_distance = float('inf')
|
||||
self.last_food_steps = 0
|
||||
self.steps_since_direction_change = 0 # Track steps since last direction change
|
||||
self.last_direction = None # Track previous direction
|
||||
self.recent_positions = []
|
||||
self.steps_in_same_direction = 0 # Track steps without direction change
|
||||
self.last_min_food_distance = float('inf') # Initialize minimum distance tracking
|
||||
|
||||
# For rendering
|
||||
self.window = None
|
||||
self.clock = None
|
||||
|
||||
self.score_milestone_rewards = {5: 10.0, 10: 20.0, 15: 30.0} # Bonus rewards at score milestones
|
||||
self.last_milestone = 0 # Track last milestone reached
|
||||
self.base_survival_penalty = -1.0 # Much larger constant penalty per step
|
||||
|
||||
# Movement control parameters
|
||||
self.max_direction_steps = 10 # Reduced from 20 to 10
|
||||
self.direction_step_increase = 2 # Reduced from 5 to 2
|
||||
self.min_safe_turns = 2 # Reduced from 3 to 2
|
||||
self.repetitive_threshold = 10 # Base threshold before repetitive penalty kicks in
|
||||
self.repetitive_scale = 1.2 # Base scale for penalty growth
|
||||
self.repetitive_wrap_scale = 2.0 # Much faster growth rate in wrap mode
|
||||
self.max_repetitive_penalty = -5.0 # Increased maximum repetitive movement penalty
|
||||
|
||||
# Initialize base rewards with adjusted values
|
||||
rewards = {
|
||||
'food': 0.0,
|
||||
'death': 0.0,
|
||||
'distance': 0.0,
|
||||
'survival': self.base_survival_penalty * 0.5,
|
||||
'direction': 0.0,
|
||||
'efficiency': 0.0,
|
||||
'progress': 0.0,
|
||||
'alignment': 0.0,
|
||||
'repetitive': 0.0
|
||||
}
|
||||
|
||||
def _get_normalized_observation(self) -> np.ndarray:
|
||||
"""
|
||||
Create a multi-channel 2D observation from the game state.
|
||||
Spatial channels encode the grid layout (snake & food),
|
||||
while extra scalar features are broadcast over additional channels.
|
||||
"""
|
||||
state = self.session.get_state()
|
||||
snake_body = state["snake_body"]
|
||||
food_position = state["food_position"]
|
||||
head = snake_body[0]
|
||||
body = snake_body
|
||||
|
||||
# --- Spatial Channels ---
|
||||
# Channel 0: Snake positions (1 for snake, 0 otherwise)
|
||||
snake_channel = np.zeros((self.size, self.size), dtype=np.float32)
|
||||
for pos in snake_body:
|
||||
snake_channel[pos[1], pos[0]] = 1.0
|
||||
assert pos[0] < self.size and pos[1] < self.size, \
|
||||
f"Snake position {pos} exceeds grid size {self.size}"
|
||||
|
||||
# Channel 1: Food position
|
||||
food_channel = np.zeros((self.size, self.size), dtype=np.float32)
|
||||
food_channel[food_position[1], food_position[0]] = 1.0
|
||||
|
||||
# --- Auxiliary Scalar Features ---
|
||||
# Compute distances and apply wrap-around adjustments
|
||||
dx = food_position[0] - head[0]
|
||||
dy = food_position[1] - head[1]
|
||||
if self.rules.wrap_around:
|
||||
if dx > self.size/2:
|
||||
wrap_dx = dx - self.size
|
||||
elif dx < -self.size/2:
|
||||
wrap_dx = dx + self.size
|
||||
else:
|
||||
wrap_dx = dx
|
||||
if dy > self.size/2:
|
||||
wrap_dy = dy - self.size
|
||||
elif dy < -self.size/2:
|
||||
wrap_dy = dy + self.size
|
||||
else:
|
||||
wrap_dy = dy
|
||||
else:
|
||||
wrap_dx = dx
|
||||
wrap_dy = dy
|
||||
|
||||
# Normalize
|
||||
dx_norm = dx / self.size
|
||||
dy_norm = dy / self.size
|
||||
wrap_dx_norm = wrap_dx / self.size
|
||||
wrap_dy_norm = wrap_dy / self.size
|
||||
|
||||
# Danger indicators (vector of 4 values)
|
||||
dangers = self._get_danger_observations_all_directions(head, body)
|
||||
|
||||
# Other auxiliary scalar features
|
||||
snake_length_norm = len(body) / (self.size**2)
|
||||
# One-hot encode current direction
|
||||
direction_one_hot = [0.0] * 4
|
||||
direction_one_hot[self.direction_to_index[state["snake_direction"]]] = 1.0
|
||||
|
||||
# One-hot encode relative food direction
|
||||
food_dir = [0.0] * 4
|
||||
if abs(dx_norm) > abs(dy_norm):
|
||||
if dx_norm > 0:
|
||||
food_dir[self.direction_to_index[Direction.RIGHT]] = 1.0
|
||||
else:
|
||||
food_dir[self.direction_to_index[Direction.LEFT]] = 1.0
|
||||
else:
|
||||
if dy_norm > 0:
|
||||
food_dir[self.direction_to_index[Direction.DOWN]] = 1.0
|
||||
else:
|
||||
food_dir[self.direction_to_index[Direction.UP]] = 1.0
|
||||
|
||||
current_time_limit = min(
|
||||
self.max_time_limit,
|
||||
self.base_time_limit + (len(body) - 1) * self.length_time_bonus
|
||||
)
|
||||
time_pressure = np.clip(self.steps_since_food / current_time_limit, 0, 1)
|
||||
wrap_mode = float(self.rules.wrap_around)
|
||||
progress = min(1.0, len(body) / 10)
|
||||
recent_food = min(1.0, self.steps_since_food / 50)
|
||||
curriculum_stage_norm = float(self.curriculum_stage) / 3
|
||||
advanced_length = float(len(body) > 5)
|
||||
manhattan_norm = min(1.0, manhattan_distance(head, food_position) / (self.size * 2))
|
||||
|
||||
# --- Create "broadcasted" channels for each scalar ---
|
||||
def broadcast_channel(value):
|
||||
return np.full((self.size, self.size), value, dtype=np.float32)
|
||||
|
||||
dx_channel = broadcast_channel(dx_norm)
|
||||
dy_channel = broadcast_channel(dy_norm)
|
||||
wrap_dx_channel = broadcast_channel(wrap_dx_norm)
|
||||
wrap_dy_channel = broadcast_channel(wrap_dy_norm)
|
||||
snake_length_chan = broadcast_channel(snake_length_norm)
|
||||
time_pressure_chan = broadcast_channel(time_pressure)
|
||||
wrap_mode_chan = broadcast_channel(wrap_mode)
|
||||
progress_chan = broadcast_channel(progress)
|
||||
recent_food_chan = broadcast_channel(recent_food)
|
||||
curriculum_chan = broadcast_channel(curriculum_stage_norm)
|
||||
advanced_length_chan = broadcast_channel(advanced_length)
|
||||
manhattan_chan = broadcast_channel(manhattan_norm)
|
||||
|
||||
# For one-hot features, create one channel per value
|
||||
direction_channels = [broadcast_channel(v) for v in direction_one_hot]
|
||||
food_dir_channels = [broadcast_channel(v) for v in food_dir]
|
||||
|
||||
# For dangers, one channel per direction (4 channels)
|
||||
danger_channels = [broadcast_channel(d) for d in dangers]
|
||||
|
||||
# --- Stack all channels ---
|
||||
# You can adjust the channel order as desired.
|
||||
channels = [
|
||||
snake_channel, # Channel 0
|
||||
food_channel, # Channel 1
|
||||
dx_channel, # Channel 2
|
||||
dy_channel, # Channel 3
|
||||
wrap_dx_channel, # Channel 4
|
||||
wrap_dy_channel, # Channel 5
|
||||
]
|
||||
channels.extend(danger_channels) # Channels 6-9
|
||||
channels.append(snake_length_chan) # Channel 10
|
||||
channels.extend(direction_channels) # Channels 11-14
|
||||
channels.extend(food_dir_channels) # Channels 15-18
|
||||
channels.append(time_pressure_chan) # Channel 19
|
||||
channels.append(wrap_mode_chan) # Channel 20
|
||||
channels.append(progress_chan) # Channel 21
|
||||
channels.append(recent_food_chan) # Channel 22
|
||||
channels.append(curriculum_chan) # Channel 23
|
||||
channels.append(advanced_length_chan) # Channel 24
|
||||
channels.append(manhattan_chan) # Channel 25
|
||||
|
||||
# Final observation: shape (num_channels, size, size)
|
||||
observation = np.stack(channels, axis=0)
|
||||
return np.clip(observation, 0, 1)
|
||||
|
||||
def _get_danger_observations_all_directions(self, head: Tuple[int, int], body: List[Tuple[int, int]]) -> List[float]:
|
||||
"""Get danger observations in all four directions."""
|
||||
dangers = [0.0] * 4 # [UP, RIGHT, DOWN, LEFT]
|
||||
|
||||
# Check each direction
|
||||
directions = [Direction.UP, Direction.RIGHT, Direction.DOWN, Direction.LEFT]
|
||||
for i, direction in enumerate(directions):
|
||||
dx, dy = direction.to_vector()
|
||||
next_pos = (head[0] + dx, head[1] + dy)
|
||||
|
||||
# Check wall collision
|
||||
if not self.rules.wrap_around:
|
||||
if (next_pos[0] < 0 or next_pos[0] >= self.size or
|
||||
next_pos[1] < 0 or next_pos[1] >= self.size):
|
||||
dangers[i] = 1.0
|
||||
continue
|
||||
else:
|
||||
next_pos = (next_pos[0] % self.size, next_pos[1] % self.size)
|
||||
|
||||
# Check self collision
|
||||
if next_pos in body[1:]:
|
||||
dangers[i] = 1.0
|
||||
|
||||
return dangers
|
||||
|
||||
def reset(self, seed=None, options=None):
|
||||
"""Reset with curriculum-based difficulty and exploration decay."""
|
||||
# Call parent reset without unpacking
|
||||
super().reset(seed=seed)
|
||||
if seed is not None:
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
|
||||
# Update exploration bonus
|
||||
self.total_episodes += 1
|
||||
self.current_exploration_bonus = max(
|
||||
self.min_exploration_bonus,
|
||||
self.initial_exploration_bonus * (self.exploration_decay ** self.total_episodes)
|
||||
)
|
||||
|
||||
# Initialize game session with curriculum-appropriate settings
|
||||
self.session = GameSession(
|
||||
self.size,
|
||||
self.size,
|
||||
self.rules
|
||||
)
|
||||
|
||||
# Reset counters
|
||||
self.steps = 0
|
||||
self.steps_since_food = 0
|
||||
self.current_time = 0
|
||||
self.last_score = 0
|
||||
self.last_distance = float('inf')
|
||||
self.last_direction = self.session.get_state()["snake_direction"] # Track initial direction
|
||||
self.recent_positions = []
|
||||
self.steps_in_same_direction = 0 # Track steps without direction change
|
||||
self.last_min_food_distance = float('inf') # Initialize minimum distance tracking
|
||||
|
||||
# Get initial observation
|
||||
observation = self._get_normalized_observation()
|
||||
info = {
|
||||
"curriculum_stage": self.curriculum_stage,
|
||||
"exploration_bonus": self.current_exploration_bonus
|
||||
}
|
||||
|
||||
return observation, info
|
||||
|
||||
def _calculate_area_penalty(self, head_pos):
|
||||
"""Calculate penalty for staying in a small area."""
|
||||
# Add current position to history
|
||||
self.recent_positions.append(head_pos)
|
||||
if len(self.recent_positions) > self.max_recent_positions:
|
||||
self.recent_positions.pop(0)
|
||||
|
||||
if len(self.recent_positions) < 10: # Need minimum history to calculate
|
||||
return 0.0
|
||||
|
||||
# Calculate bounding box of recent positions
|
||||
x_coords = [p[0] for p in self.recent_positions]
|
||||
y_coords = [p[1] for p in self.recent_positions]
|
||||
area_width = max(x_coords) - min(x_coords) + 1
|
||||
area_height = max(y_coords) - min(y_coords) + 1
|
||||
area = area_width * area_height
|
||||
|
||||
# Calculate unique positions visited
|
||||
unique_positions = len(set(self.recent_positions))
|
||||
|
||||
# Penalize small areas and repeated positions
|
||||
area_penalty = 0.0
|
||||
if area < 9: # 3x3 grid or smaller
|
||||
area_penalty = -0.2 * (9 - area) / 8 # Max penalty -0.2 for 1x1 area
|
||||
|
||||
# Additional penalty for revisiting same positions frequently
|
||||
repetition_penalty = -0.3 * (1 - unique_positions / len(self.recent_positions))
|
||||
|
||||
return area_penalty + repetition_penalty
|
||||
|
||||
def step(self, action):
|
||||
"""Improved reward structure with length-aware exploration."""
|
||||
self.steps += 1
|
||||
self.steps_since_food += 1
|
||||
|
||||
# Calculate current time limit based on snake length
|
||||
snake_length = len(self.session.get_state()["snake_body"])
|
||||
current_time_limit = min(
|
||||
self.max_time_limit,
|
||||
self.base_time_limit + (snake_length - 1) * self.length_time_bonus
|
||||
)
|
||||
|
||||
# Get state before action
|
||||
prev_state = self.session.get_state()
|
||||
prev_head = prev_state["snake_head"]
|
||||
prev_food_dist = manhattan_distance(prev_head, prev_state["food_position"])
|
||||
if self.rules.wrap_around:
|
||||
prev_food_dist = min(prev_food_dist,
|
||||
manhattan_distance_wrap(prev_head, prev_state["food_position"], self.size))
|
||||
|
||||
# Convert action to Direction and apply
|
||||
direction = self.action_to_direction[int(action)]
|
||||
|
||||
# Update game state
|
||||
state, base_reward, done = self.session.step(direction, self.current_time)
|
||||
|
||||
# Initialize reward components
|
||||
rewards = {
|
||||
'food': 0.0,
|
||||
'death': 0.0,
|
||||
'distance': 0.0,
|
||||
'survival': 0.0,
|
||||
'milestone': 0.0,
|
||||
'efficiency': 0.0,
|
||||
'exploration': 0.0,
|
||||
'safety': 0.0,
|
||||
'timeout': 0.0,
|
||||
'direction': 0.0,
|
||||
'repetitive': 0.0
|
||||
}
|
||||
|
||||
# Get current state info
|
||||
curr_head = state["snake_body"][0]
|
||||
curr_food = state["food_position"]
|
||||
curr_food_dist = manhattan_distance(curr_head, curr_food)
|
||||
if self.rules.wrap_around:
|
||||
curr_food_dist = min(curr_food_dist,
|
||||
manhattan_distance_wrap(curr_head, curr_food, self.size))
|
||||
|
||||
# Update current time
|
||||
self.current_time += self.session.move_cooldown
|
||||
|
||||
# Food reward with exponential scaling based on snake length
|
||||
food_eaten = state["score"] > prev_state["score"]
|
||||
if food_eaten:
|
||||
# Reset timer when food is eaten
|
||||
self.steps_since_food = 0
|
||||
|
||||
# Calculate exponential food reward
|
||||
base_food_reward = self.reward_scales['food']
|
||||
length_multiplier = 1.2 ** (len(state["snake_body"]) - 1) # 20% increase per length
|
||||
rewards['food'] = base_food_reward * length_multiplier
|
||||
|
||||
# Add milestone rewards for achieving certain scores
|
||||
if state["score"] in [5, 10, 15, 20]:
|
||||
rewards['milestone'] = self.reward_scales['milestone'] * (state["score"] / 5)
|
||||
else:
|
||||
# Only apply survival penalty if we haven't eaten food in a while
|
||||
if self.steps_since_food > current_time_limit / 2:
|
||||
penalty_factor = (self.steps_since_food - current_time_limit/2) / (current_time_limit/2)
|
||||
rewards['survival'] = self.reward_scales['survival'] * penalty_factor
|
||||
|
||||
# Check if direction changed and update counters
|
||||
direction_changed = direction != self.last_direction
|
||||
if direction_changed:
|
||||
self.steps_in_same_direction = 0
|
||||
else:
|
||||
self.steps_in_same_direction += 1
|
||||
# Apply growing penalty for repetitive movement
|
||||
if self.steps_in_same_direction > (self.repetitive_threshold - 5 if self.rules.wrap_around else self.repetitive_threshold):
|
||||
excess_steps = self.steps_in_same_direction - (self.repetitive_threshold - 5 if self.rules.wrap_around else self.repetitive_threshold)
|
||||
# Use much faster growth rate in wrap-around mode
|
||||
growth_rate = self.repetitive_wrap_scale if self.rules.wrap_around else self.repetitive_scale
|
||||
repetitive_penalty = self.reward_scales['repetitive'] * (growth_rate ** excess_steps)
|
||||
|
||||
# Apply additional multiplier in wrap-around mode that grows with steps
|
||||
if self.rules.wrap_around:
|
||||
wrap_multiplier = 1.0 + (excess_steps * 0.5) # Multiplier grows with each step
|
||||
repetitive_penalty *= wrap_multiplier
|
||||
rewards['repetitive'] = repetitive_penalty # No maximum cap - let it grow unbounded in wrap-around mode
|
||||
|
||||
else:
|
||||
# Cap the penalty when not in wrap-around mode
|
||||
rewards['repetitive'] = max(repetitive_penalty, self.max_repetitive_penalty)
|
||||
|
||||
|
||||
self.last_direction = direction
|
||||
|
||||
# Check for timeout or stuck in loop - only if we haven't just eaten food
|
||||
if (self.steps_since_food >= current_time_limit and not food_eaten):
|
||||
done = True
|
||||
rewards['timeout'] = self.reward_scales['timeout']
|
||||
# Scale timeout penalty based on distance to food
|
||||
if curr_food_dist < 5: # Harsher penalty for timing out near food
|
||||
rewards['timeout'] *= 1.5
|
||||
# Additional penalty for getting stuck in a loop
|
||||
if self.steps_in_same_direction >= 30: # Also update this threshold
|
||||
rewards['timeout'] *= 1.2
|
||||
|
||||
# Death penalty - calculate AFTER food rewards
|
||||
if done and not self.steps_since_food >= current_time_limit:
|
||||
base_death_penalty = self.reward_scales['death']
|
||||
|
||||
# Check if death was due to self-collision
|
||||
if curr_head in state["snake_body"][1:]:
|
||||
# Base multiplier for self-collision
|
||||
self_collision_multiplier = 2.0
|
||||
|
||||
# Additional penalty scaling with score
|
||||
# At score 5: 2.5x penalty
|
||||
# At score 10: 3.0x penalty
|
||||
# At score 15: 3.5x penalty
|
||||
score_multiplier = 1.0 + (state["score"] / 10)
|
||||
|
||||
# Apply both multipliers
|
||||
rewards['death'] = base_death_penalty * self_collision_multiplier * score_multiplier
|
||||
|
||||
# Additional penalty if we died near food
|
||||
if curr_food_dist < 5:
|
||||
rewards['death'] *= 1.5 # Even harsher if we collide with ourselves near food
|
||||
else:
|
||||
# For wall collisions, keep original scaling but make it less punishing at higher scores
|
||||
rewards['death'] = base_death_penalty * (1.0 - min(0.5, state["score"] / 20))
|
||||
|
||||
# Add an immediate negative reward for losing potential score
|
||||
potential_loss_penalty = -0.5 * state["score"] # Larger penalty for dying with higher scores
|
||||
rewards['death'] += potential_loss_penalty
|
||||
|
||||
# Track minimum distance to food and check for near misses
|
||||
if curr_food_dist < self.last_min_food_distance:
|
||||
self.last_min_food_distance = curr_food_dist
|
||||
elif curr_food_dist > self.last_min_food_distance:
|
||||
# We're moving away from our closest approach
|
||||
if self.last_min_food_distance <= self.near_miss_threshold:
|
||||
# We got very close but missed
|
||||
rewards['distance'] += self.reward_scales['near_miss']
|
||||
self.last_min_food_distance = float('inf') # Reset tracking
|
||||
|
||||
# Progressive distance reward - more reward for getting closer when near food
|
||||
distance_change = prev_food_dist - curr_food_dist
|
||||
if curr_food_dist < 5: # When close to food
|
||||
distance_multiplier = 2.0 # Double the reward/penalty
|
||||
else:
|
||||
distance_multiplier = 1.0
|
||||
rewards['distance'] = distance_change * self.reward_scales['distance'] * distance_multiplier
|
||||
|
||||
# Calculate direction penalty
|
||||
if curr_food_dist <= 5: # Only apply when close to food
|
||||
# Calculate optimal direction to food
|
||||
dx = curr_food[0] - curr_head[0]
|
||||
dy = curr_food[1] - curr_head[1]
|
||||
|
||||
if self.rules.wrap_around:
|
||||
# Adjust for wrap-around
|
||||
if dx > self.size/2: dx -= self.size
|
||||
elif dx < -self.size/2: dx += self.size
|
||||
if dy > self.size/2: dy -= self.size
|
||||
elif dy < -self.size/2: dy += self.size
|
||||
|
||||
# Determine optimal direction(s)
|
||||
optimal_directions = []
|
||||
if abs(dx) > abs(dy):
|
||||
if dx > 0: optimal_directions.append(Direction.RIGHT)
|
||||
elif dx < 0: optimal_directions.append(Direction.LEFT)
|
||||
if abs(dy) >= abs(dx):
|
||||
if dy > 0: optimal_directions.append(Direction.DOWN)
|
||||
elif dy < 0: optimal_directions.append(Direction.UP)
|
||||
|
||||
# Base penalty scales with proximity (closer = higher penalty)
|
||||
base_penalty = (6 - curr_food_dist) * 0.1 # Scales from 0.1 to 0.5
|
||||
|
||||
if direction not in optimal_directions:
|
||||
# Check if we're moving directly away from food
|
||||
opposite_dirs = {
|
||||
Direction.UP: Direction.DOWN,
|
||||
Direction.DOWN: Direction.UP,
|
||||
Direction.LEFT: Direction.RIGHT,
|
||||
Direction.RIGHT: Direction.LEFT
|
||||
}
|
||||
optimal_opposites = [opposite_dirs[d] for d in optimal_directions]
|
||||
|
||||
if direction in optimal_opposites:
|
||||
# Double penalty for moving directly away from food
|
||||
rewards['direction'] = -2 * base_penalty
|
||||
else:
|
||||
# Regular penalty for suboptimal direction
|
||||
rewards['direction'] = -base_penalty
|
||||
elif curr_food_dist <= 2: # Small reward for correct direction when very close
|
||||
rewards['direction'] = 0.1
|
||||
|
||||
# Efficiency reward - encourage purposeful movement
|
||||
if not done and self.steps_since_food < 50: # Only apply when actively hunting
|
||||
rewards['efficiency'] = self.reward_scales['efficiency'] * (1.0 - self.steps_since_food / 50)
|
||||
|
||||
# Calculate final reward - simple sum of components
|
||||
reward = sum(rewards.values())
|
||||
|
||||
# Get observation
|
||||
observation = self._get_normalized_observation()
|
||||
|
||||
# No more episode-based truncation
|
||||
truncated = False
|
||||
|
||||
# Include reward components and exploration info in info dict for monitoring
|
||||
info = {
|
||||
"score": state["score"],
|
||||
"reward_components": rewards,
|
||||
"exploration_bonus": self.current_exploration_bonus,
|
||||
"direction_changed": direction_changed,
|
||||
"steps_in_same_direction": self.steps_in_same_direction,
|
||||
"snake_length": snake_length,
|
||||
"steps": self.steps # Include total steps for monitoring
|
||||
}
|
||||
|
||||
return observation, reward, done, truncated, info
|
||||
|
||||
def render(self):
|
||||
"""Render the environment."""
|
||||
if self.window is None and self.render_mode == "human":
|
||||
pygame.init()
|
||||
self.window = pygame.display.set_mode((self.window_width, self.window_height))
|
||||
pygame.display.set_caption("Snake Environment")
|
||||
self.clock = pygame.time.Clock()
|
||||
|
||||
if self.window is not None:
|
||||
self.window.fill((0, 0, 0))
|
||||
|
||||
# Draw game area
|
||||
self.game_area.draw(self.window)
|
||||
|
||||
# Draw game session
|
||||
self.session.render(self.window, self.game_area)
|
||||
|
||||
# Draw score
|
||||
font = pygame.font.Font(None, 36)
|
||||
score_text = f"Score: {self.session.score}"
|
||||
score_surface = font.render(score_text, True, (0, 255, 0))
|
||||
score_rect = score_surface.get_rect(midtop=(self.window_width // 2, 20))
|
||||
self.window.blit(score_surface, score_rect)
|
||||
|
||||
pygame.display.flip()
|
||||
self.clock.tick(self.metadata["render_fps"])
|
||||
|
||||
if self.render_mode == "rgb_array":
|
||||
return np.transpose(
|
||||
np.array(pygame.surfarray.pixels3d(self.window)),
|
||||
axes=(1, 0, 2)
|
||||
)
|
||||
|
||||
def close(self):
|
||||
"""Close the environment."""
|
||||
if self.window is not None:
|
||||
pygame.quit()
|
||||
self.window = None
|
||||
|
||||
def get_state(self) -> Dict:
|
||||
"""Get the current game state."""
|
||||
state = self.session.get_state()
|
||||
state['steps_since_food'] = self.steps_since_food
|
||||
|
||||
# Calculate and include current time limit
|
||||
snake_length = len(state["snake_body"])
|
||||
current_time_limit = min(
|
||||
self.max_time_limit,
|
||||
self.base_time_limit + (snake_length - 1) * self.length_time_bonus
|
||||
)
|
||||
state['current_time_limit'] = current_time_limit
|
||||
|
||||
return state
|
||||
|
||||
def _get_adjacent_positions(self, position: Tuple[int, int]) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Get all adjacent positions to the given position.
|
||||
|
||||
Args:
|
||||
position: Current position (x, y)
|
||||
|
||||
Returns:
|
||||
List of adjacent positions [(x, y), ...]
|
||||
|
||||
Raises:
|
||||
ValueError: If position is None or not a tuple of 2 integers
|
||||
ValueError: If position coordinates are not integers
|
||||
"""
|
||||
if not position or not isinstance(position, tuple) or len(position) != 2:
|
||||
raise ValueError("Position must be a tuple of 2 coordinates")
|
||||
|
||||
try:
|
||||
x, y = int(position[0]), int(position[1])
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError("Position coordinates must be integers")
|
||||
|
||||
# Get positions in all 4 directions (up, right, down, left)
|
||||
adjacent = [
|
||||
(x, y - 1), # up
|
||||
(x + 1, y), # right
|
||||
(x, y + 1), # down
|
||||
(x - 1, y) # left
|
||||
]
|
||||
|
||||
# Validate size attribute exists and is positive integer
|
||||
if not hasattr(self, 'size') or not isinstance(self.size, int) or self.size <= 0:
|
||||
raise ValueError("Environment size must be a positive integer")
|
||||
|
||||
# If wrap-around is enabled, adjust out-of-bounds positions
|
||||
if hasattr(self, 'rules') and hasattr(self.rules, 'wrap_around') and self.rules.wrap_around:
|
||||
adjacent = [
|
||||
(pos[0] % self.size, pos[1] % self.size)
|
||||
for pos in adjacent
|
||||
]
|
||||
return adjacent
|
||||
|
||||
# Otherwise, only return positions that are within bounds
|
||||
return [
|
||||
(pos[0], pos[1]) for pos in adjacent
|
||||
if 0 <= pos[0] < self.size and 0 <= pos[1] < self.size
|
||||
]
|
||||
|
||||
def _update_curriculum(self, score: int, done: bool) -> None:
|
||||
"""Update curriculum stage based on agent's performance."""
|
||||
if done:
|
||||
if score >= self.stage_requirements[self.curriculum_stage]:
|
||||
self.consecutive_successes += 1
|
||||
else:
|
||||
self.consecutive_successes = 0
|
||||
|
||||
# Check for stage advancement
|
||||
if (self.curriculum_stage < 3 and
|
||||
self.consecutive_successes >= self.success_threshold[self.curriculum_stage]):
|
||||
self.curriculum_stage += 1
|
||||
self.consecutive_successes = 0
|
||||
# Increase episode length with curriculum stage
|
||||
self.max_steps = self.base_max_steps * (1 + self.curriculum_stage * 0.5)
|
||||
|
||||
def _get_curriculum_rules(self) -> GameRules:
|
||||
"""Get game rules appropriate for current curriculum stage."""
|
||||
rules = GameRules()
|
||||
|
||||
# Adjust rules based on curriculum stage
|
||||
if self.curriculum_stage == 0:
|
||||
rules.wrap_around = True # Make it easier to avoid walls
|
||||
rules.speed_increase = False
|
||||
rules.initial_move_cooldown = 200 # Much slower movement
|
||||
elif self.curriculum_stage == 1:
|
||||
rules.wrap_around = True
|
||||
rules.speed_increase = False # Still no speed increase
|
||||
rules.initial_move_cooldown = 150
|
||||
elif self.curriculum_stage == 2:
|
||||
rules.wrap_around = True # Keep wrap-around until final stage
|
||||
rules.speed_increase = True
|
||||
rules.initial_move_cooldown = 120
|
||||
else: # Stage 3
|
||||
rules.wrap_around = False
|
||||
rules.speed_increase = True
|
||||
rules.initial_move_cooldown = 100
|
||||
|
||||
return rules
|
||||
|
||||
# Register the environment with Gymnasium
|
||||
from gymnasium.envs.registration import register
|
||||
|
||||
register(
|
||||
id='Snake-v0',
|
||||
entry_point='src.ai.environment:SnakeEnv',
|
||||
kwargs={
|
||||
'size': 30,
|
||||
'render_mode': None,
|
||||
'difficulty': 'easy'
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,93 @@
|
||||
import gym
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from stable_baselines3.common.torch_layers import BaseFeaturesExtractor
|
||||
|
||||
|
||||
class CustomRecurrentCNN(BaseFeaturesExtractor):
|
||||
"""
|
||||
A simple CNN with an LSTM layer added on top. This enables the network to keep an internal
|
||||
memory of past observations so that it can adapt to changing rules/environment dynamics.
|
||||
"""
|
||||
def __init__(self, observation_space, features_dim=512):
|
||||
# Initialize with a features_dim that matches the LSTM's output.
|
||||
super(CustomRecurrentCNN, self).__init__(observation_space, features_dim)
|
||||
|
||||
# Calculate the expected CNN output size first
|
||||
n_input_channels = observation_space.shape[0] # Number of input channels
|
||||
|
||||
self.cnn = nn.Sequential(
|
||||
# Layer 1: (n_channels, 30, 30) -> (32, 15, 15)
|
||||
nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=2, padding=1),
|
||||
nn.ReLU(),
|
||||
|
||||
# Layer 2: (32, 15, 15) -> (64, 8, 8)
|
||||
nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1),
|
||||
nn.ReLU(),
|
||||
|
||||
# Layer 3: (64, 8, 8) -> (64, 4, 4)
|
||||
nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1),
|
||||
nn.ReLU(),
|
||||
|
||||
nn.Flatten()
|
||||
)
|
||||
|
||||
# Calculate CNN output dimension using a dummy forward pass
|
||||
with torch.no_grad():
|
||||
dummy_input = torch.zeros(1, *observation_space.shape)
|
||||
cnn_output = self.cnn(dummy_input)
|
||||
self._cnn_output_dim = cnn_output.shape[1] # Should be 64 * 4 * 4 = 1024
|
||||
|
||||
# Add linear layer to reduce CNN output to desired feature dimension
|
||||
self.fc = nn.Linear(self._cnn_output_dim, 256)
|
||||
|
||||
# LSTM layer
|
||||
self.lstm = nn.LSTM(256, features_dim, batch_first=True)
|
||||
|
||||
def forward(self, observations):
|
||||
# Pass through CNN: (batch, channels, height, width) -> (batch, cnn_features)
|
||||
cnn_out = self.cnn(observations)
|
||||
|
||||
# Pass through linear layer: (batch, cnn_features) -> (batch, 256)
|
||||
fc_out = self.fc(cnn_out)
|
||||
|
||||
# Add time dimension for LSTM: (batch, 256) -> (batch, 1, 256)
|
||||
lstm_input = fc_out.unsqueeze(1)
|
||||
|
||||
# LSTM: (batch, 1, 256) -> (batch, 1, features_dim)
|
||||
lstm_out, _ = self.lstm(lstm_input)
|
||||
|
||||
# Remove time dimension: (batch, 1, features_dim) -> (batch, features_dim)
|
||||
return lstm_out.squeeze(1)
|
||||
|
||||
|
||||
class CustomCNN(BaseFeaturesExtractor):
|
||||
"""
|
||||
Custom feature extractor for the snake environment.
|
||||
Uses fully connected layers since our input is a 1D vector of normalized features.
|
||||
"""
|
||||
|
||||
def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):
|
||||
super().__init__(observation_space, features_dim)
|
||||
self.input_dim = int(np.prod(observation_space.shape))
|
||||
|
||||
self.cnn = nn.Sequential(
|
||||
nn.Linear(self.input_dim, 512),
|
||||
nn.ReLU(),
|
||||
nn.BatchNorm1d(512),
|
||||
nn.Linear(512, 512),
|
||||
nn.ReLU(),
|
||||
nn.BatchNorm1d(512),
|
||||
nn.Linear(512, features_dim),
|
||||
nn.ReLU(),
|
||||
)
|
||||
|
||||
def forward(self, observations: torch.Tensor) -> torch.Tensor:
|
||||
# Handle both single and batch observations
|
||||
if len(observations.shape) == 1:
|
||||
observations = observations.unsqueeze(0)
|
||||
|
||||
# Ensure the input dimension matches what we expect
|
||||
flat_obs = observations.reshape(observations.shape[0], self.input_dim)
|
||||
return self.cnn(flat_obs)
|
||||
+423
@@ -0,0 +1,423 @@
|
||||
"""
|
||||
Training script for Snake AI using Stable Baselines3.
|
||||
|
||||
This script sets up and trains RL agents using the SnakeEnv environment.
|
||||
It supports multiple difficulty levels and provides real-time visualization
|
||||
of the training process.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import numpy as np
|
||||
import torch
|
||||
from stable_baselines3 import PPO
|
||||
from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv
|
||||
from stable_baselines3.common.callbacks import BaseCallback
|
||||
from stable_baselines3.common.utils import set_random_seed
|
||||
from stable_baselines3.common.monitor import Monitor
|
||||
import gymnasium as gym
|
||||
import torch.nn as nn
|
||||
import argparse
|
||||
from queue import Queue
|
||||
from threading import Thread
|
||||
from typing import Dict, Any, Optional
|
||||
import json
|
||||
from queue import Empty
|
||||
|
||||
from src.ai.environment import SnakeEnv
|
||||
from src.ai.visualize import run_dashboard
|
||||
from src.ai.model_networks import CustomRecurrentCNN, CustomCNN
|
||||
|
||||
class VisualizationCallback(BaseCallback):
|
||||
"""Callback for updating the training visualization."""
|
||||
|
||||
def __init__(self, viz_queue: Queue, total_timesteps: int, eval_freq: int = 1000, n_steps: int = 512, n_envs: int = 32, initial_timesteps: int = 0):
|
||||
super().__init__()
|
||||
self.viz_queue = viz_queue
|
||||
self.total_timesteps = total_timesteps
|
||||
self.eval_freq = eval_freq
|
||||
self.n_steps = n_steps
|
||||
self.n_envs = n_envs
|
||||
self.initial_timesteps = initial_timesteps # Store initial timesteps
|
||||
|
||||
# Training metrics
|
||||
self.episode_rewards = []
|
||||
self.episode_lengths = []
|
||||
|
||||
# Evaluation metrics
|
||||
self.eval_scores = []
|
||||
self.high_score = 0
|
||||
self.recent_high_score = 0
|
||||
|
||||
# Current episode tracking
|
||||
self.current_reward = 0
|
||||
self.current_length = 0
|
||||
|
||||
# Evaluation games (one with wrap, one without)
|
||||
self.eval_envs = [
|
||||
Monitor(SnakeEnv(render_mode="rgb_array")) for _ in range(2)
|
||||
]
|
||||
self.eval_envs[0].unwrapped.rules.wrap_around = False
|
||||
self.eval_envs[1].unwrapped.rules.wrap_around = True
|
||||
|
||||
self.eval_obs = [None, None]
|
||||
self.eval_done = [True, True]
|
||||
self.eval_scores_current = [0, 0]
|
||||
|
||||
# Real-time demo environment
|
||||
self.demo_env = Monitor(SnakeEnv(render_mode="rgb_array"))
|
||||
self.demo_env.unwrapped.rules.wrap_around = True
|
||||
self.demo_obs = None
|
||||
self.demo_done = True
|
||||
self.last_demo_time = 0
|
||||
self.demo_speed = 50
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
"""Update visualization on each step."""
|
||||
try:
|
||||
current_time = time.time() * 1000 # Convert to milliseconds
|
||||
|
||||
# Update episode tracking
|
||||
self.current_length += 1
|
||||
reward = self.locals.get("rewards", [0])[0]
|
||||
self.current_reward += reward
|
||||
|
||||
# Get action probabilities and value estimate for demo game
|
||||
if self.demo_obs is not None:
|
||||
# Ensure proper tensor shape (batch_size, obs_dim)
|
||||
demo_obs_tensor = torch.as_tensor(self.demo_obs).float()
|
||||
if len(demo_obs_tensor.shape) == 1:
|
||||
demo_obs_tensor = demo_obs_tensor.unsqueeze(0)
|
||||
|
||||
# Move tensor to same device as model's policy
|
||||
device = self.model.policy.device
|
||||
demo_obs_tensor = demo_obs_tensor.to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
# Get action distribution and value estimate
|
||||
dist = self.model.policy.get_distribution(demo_obs_tensor)
|
||||
action_probs = dist.distribution.probs
|
||||
value_estimate = self.model.policy.predict_values(demo_obs_tensor)
|
||||
|
||||
# Move results back to CPU and get first item (since we added batch dimension)
|
||||
action_probs = action_probs[0].cpu().numpy()
|
||||
value_estimate = value_estimate[0].cpu().numpy()
|
||||
else:
|
||||
action_probs = np.zeros(4)
|
||||
value_estimate = np.array([0.0])
|
||||
|
||||
# Check if episode ended
|
||||
dones = self.locals.get("dones", [False])
|
||||
if any(dones):
|
||||
self.episode_rewards.append(self.current_reward)
|
||||
self.episode_lengths.append(self.current_length)
|
||||
self.current_reward = 0
|
||||
self.current_length = 0
|
||||
|
||||
# Run evaluation games (keep these deterministic as they're meant to show best performance)
|
||||
for i, (env, obs, done) in enumerate(zip(self.eval_envs, self.eval_obs, self.eval_done)):
|
||||
if done:
|
||||
if obs is not None: # If this wasn't the first reset
|
||||
self.eval_scores.append(self.eval_scores_current[i])
|
||||
# Update high scores based on evaluation performance
|
||||
self.high_score = max(self.high_score, self.eval_scores_current[i])
|
||||
self.recent_high_score = max(self.recent_high_score, self.eval_scores_current[i])
|
||||
self.eval_obs[i] = env.reset()[0]
|
||||
self.eval_done[i] = False
|
||||
self.eval_scores_current[i] = 0
|
||||
|
||||
eval_action, _ = self.model.predict(self.eval_obs[i], deterministic=True)
|
||||
eval_action = eval_action.item() if hasattr(eval_action, 'item') else eval_action
|
||||
self.eval_obs[i], _, done, truncated, info = env.step(eval_action)
|
||||
self.eval_done[i] = done or truncated
|
||||
# Update current evaluation score
|
||||
self.eval_scores_current[i] = info.get("score", 0)
|
||||
|
||||
# Run real-time demo game
|
||||
if current_time - self.last_demo_time >= self.demo_speed:
|
||||
if self.demo_done:
|
||||
self.demo_obs = self.demo_env.reset()[0]
|
||||
self.demo_done = False
|
||||
|
||||
demo_action, _ = self.model.predict(self.demo_obs, deterministic=True)
|
||||
demo_action = demo_action.item() if hasattr(demo_action, 'item') else demo_action
|
||||
self.demo_obs, _, done, truncated, info = self.demo_env.step(demo_action)
|
||||
self.demo_done = done or truncated
|
||||
self.last_demo_time = current_time
|
||||
|
||||
# Get current states from actual training environments
|
||||
# Sample one no-wrap and one wrap environment from the training environments
|
||||
no_wrap_idx = 0 # First half are no-wrap
|
||||
wrap_idx = self.n_envs // 2 # Second half are wrap
|
||||
|
||||
training_state_1 = self.training_env.get_attr("session")[no_wrap_idx].get_state()
|
||||
training_state_1["wrap_around"] = False
|
||||
|
||||
training_state_2 = self.training_env.get_attr("session")[wrap_idx].get_state()
|
||||
training_state_2["wrap_around"] = True
|
||||
|
||||
eval_state_1 = self.eval_envs[0].unwrapped.session.get_state()
|
||||
eval_state_1["wrap_around"] = self.eval_envs[0].unwrapped.rules.wrap_around
|
||||
|
||||
eval_state_2 = self.eval_envs[1].unwrapped.session.get_state()
|
||||
eval_state_2["wrap_around"] = self.eval_envs[1].unwrapped.rules.wrap_around
|
||||
|
||||
demo_state = self.demo_env.unwrapped.session.get_state()
|
||||
demo_state["wrap_around"] = self.demo_env.unwrapped.rules.wrap_around
|
||||
|
||||
# Update viz_data
|
||||
viz_data = {
|
||||
'training_state': [training_state_1, training_state_2],
|
||||
'eval_states': [eval_state_1, eval_state_2],
|
||||
'demo_state': demo_state,
|
||||
'weights_info': {
|
||||
'action_probs': action_probs.tolist() if 'action_probs' in locals() else [0.25] * 4,
|
||||
'value_estimate': float(value_estimate[0]) if 'value_estimate' in locals() else 0.0,
|
||||
'action_labels': ['Up', 'Right', 'Down', 'Left']
|
||||
},
|
||||
'training_info': {
|
||||
'total_timesteps': self.num_timesteps,
|
||||
'initial_timesteps': self.initial_timesteps, # Add initial timesteps
|
||||
'target_timesteps': self.total_timesteps,
|
||||
'episode_reward': float(self.current_reward),
|
||||
'mean_reward': float(np.mean(self.episode_rewards[-100:])) if self.episode_rewards else 0.0,
|
||||
'episode_length': int(self.current_length),
|
||||
'mean_length': float(np.mean(self.episode_lengths[-100:])) if self.episode_lengths else 0.0,
|
||||
'mean_eval_score': float(np.mean(self.eval_scores[-100:])) if self.eval_scores else 0.0,
|
||||
'rewards_history': self.episode_rewards[-1000:],
|
||||
'lengths_history': self.episode_lengths[-1000:],
|
||||
'eval_scores_history': self.eval_scores[-1000:],
|
||||
'high_score': self.high_score,
|
||||
'recent_high_score': self.recent_high_score
|
||||
}
|
||||
}
|
||||
|
||||
# Send update to visualization
|
||||
self.viz_queue.put(('update', viz_data))
|
||||
|
||||
# Reset recent high score periodically
|
||||
if self.num_timesteps % (self.n_steps * self.n_envs) == 0:
|
||||
print(f"\nEvaluation High Score: {self.high_score}")
|
||||
print(f"Recent Evaluation High Score: {self.recent_high_score}")
|
||||
self.recent_high_score = 0
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error in visualization callback: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return True
|
||||
|
||||
def make_env(seed: int = 0, wrap_walls: bool = False) -> gym.Env:
|
||||
"""Create a training environment.
|
||||
|
||||
Args:
|
||||
seed: Random seed
|
||||
wrap_walls: Whether snake can wrap around walls
|
||||
"""
|
||||
def _init() -> gym.Env:
|
||||
env = SnakeEnv(render_mode=None)
|
||||
env = Monitor(env)
|
||||
# Set wrap-around mode
|
||||
env.unwrapped.rules.wrap_around = wrap_walls
|
||||
env.reset(seed=seed)
|
||||
return env
|
||||
|
||||
set_random_seed(seed)
|
||||
return _init
|
||||
|
||||
def train_model(
|
||||
total_timesteps: int,
|
||||
viz_queue: Queue,
|
||||
model_path: Optional[str] = None,
|
||||
cuda: bool = True,
|
||||
learning_rate: float = 3e-4, # Standard PPO learning rate
|
||||
batch_size: int = 256, # More reasonable batch size
|
||||
n_envs: int = 16, # Balanced number of environments
|
||||
n_steps: int = 2048 # Standard PPO steps
|
||||
) -> None:
|
||||
"""
|
||||
Train the snake AI model.
|
||||
|
||||
Args:
|
||||
total_timesteps: Number of timesteps to train for
|
||||
viz_queue: Queue for sending visualization updates
|
||||
model_path: Path to save/load the model
|
||||
cuda: Whether to use CUDA for training
|
||||
learning_rate: Learning rate for the optimizer
|
||||
batch_size: Batch size for training
|
||||
n_envs: Number of parallel environments
|
||||
n_steps: Number of steps per environment
|
||||
"""
|
||||
try:
|
||||
# Set up environments - half with wrap, half without
|
||||
envs = []
|
||||
for i in range(n_envs):
|
||||
wrap_walls = i >= n_envs // 2 # Half the envs with wrap-around
|
||||
envs.append(make_env(i, wrap_walls))
|
||||
|
||||
env = SubprocVecEnv(envs)
|
||||
|
||||
# PPO hyperparameters - adjusted for more stable learning
|
||||
ppo_params = dict(
|
||||
learning_rate=1e-4, # Slightly higher learning rate
|
||||
n_steps=1024, # Shorter horizon for faster updates
|
||||
batch_size=128, # Smaller batch size for more frequent updates
|
||||
n_epochs=4, # Balanced number of epochs
|
||||
gamma=0.98, # Slightly lower discount for more immediate rewards
|
||||
gae_lambda=0.9, # Lower GAE lambda for more immediate advantage estimates
|
||||
clip_range=0.1, # Smaller clip range for more conservative updates
|
||||
clip_range_vf=0.1, # Also clip value function
|
||||
normalize_advantage=True,
|
||||
ent_coef=0.2, # Moderate entropy for exploration
|
||||
vf_coef=0.8, # Balanced value function coefficient
|
||||
max_grad_norm=0.3, # More conservative gradient clipping
|
||||
target_kl=None,
|
||||
verbose=1,
|
||||
device="cuda" if cuda and torch.cuda.is_available() else "cpu"
|
||||
)
|
||||
|
||||
# Create or load model with updated architecture
|
||||
policy_kwargs = dict(
|
||||
features_extractor_class=CustomRecurrentCNN, # Use the new recurrent extractor
|
||||
features_extractor_kwargs=dict(features_dim=512), # You can adjust features_dim as needed
|
||||
net_arch=dict(
|
||||
pi=[512, 256, 128], # Policy network layers
|
||||
vf=[512, 256, 128] # Value network layers
|
||||
),
|
||||
activation_fn=nn.ReLU,
|
||||
normalize_images=False
|
||||
)
|
||||
|
||||
# Track initial timesteps for progress tracking
|
||||
initial_timesteps = 0
|
||||
|
||||
if model_path and os.path.exists(model_path):
|
||||
print(f"Loading model from {model_path}")
|
||||
try:
|
||||
model = PPO.load(
|
||||
model_path,
|
||||
env=env,
|
||||
custom_objects={"policy_kwargs": policy_kwargs},
|
||||
**ppo_params
|
||||
)
|
||||
initial_timesteps = model.num_timesteps
|
||||
print(f"Loaded model with {initial_timesteps:,} timesteps of training")
|
||||
except Exception as e:
|
||||
print(f"Failed to load model with error: {e}")
|
||||
print("Creating new model instead")
|
||||
model = PPO("CnnPolicy", env, policy_kwargs=policy_kwargs, **ppo_params)
|
||||
else:
|
||||
print("Creating new model")
|
||||
model = PPO("CnnPolicy", env, policy_kwargs=policy_kwargs, **ppo_params)
|
||||
|
||||
# Set up visualization callback with initial timesteps
|
||||
viz_callback = VisualizationCallback(
|
||||
viz_queue=viz_queue,
|
||||
total_timesteps=total_timesteps,
|
||||
eval_freq=1000,
|
||||
n_steps=n_steps,
|
||||
n_envs=n_envs,
|
||||
initial_timesteps=initial_timesteps # Pass initial timesteps to callback
|
||||
)
|
||||
|
||||
# Training loop with stop check
|
||||
should_stop = False
|
||||
timesteps_per_batch = n_steps * n_envs
|
||||
num_batches = total_timesteps // timesteps_per_batch
|
||||
|
||||
for i in range(num_batches):
|
||||
if should_stop:
|
||||
break
|
||||
|
||||
# Train for one batch
|
||||
model.learn(
|
||||
total_timesteps=timesteps_per_batch,
|
||||
callback=viz_callback,
|
||||
progress_bar=True,
|
||||
reset_num_timesteps=False
|
||||
)
|
||||
|
||||
# Check for stop signal from visualization
|
||||
try:
|
||||
while True: # Process all pending messages
|
||||
msg_type, _ = viz_queue.get_nowait()
|
||||
if msg_type == 'stop_training':
|
||||
should_stop = True
|
||||
break
|
||||
except Empty:
|
||||
pass
|
||||
|
||||
# Save model if training completed or interrupted
|
||||
if model_path:
|
||||
print(f"Saving model to {model_path}")
|
||||
model.save(model_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error in training process: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
# Clean up
|
||||
env.close()
|
||||
# Signal visualization to stop if we haven't already
|
||||
viz_queue.put(('stop', None))
|
||||
|
||||
def main():
|
||||
"""Main training script."""
|
||||
parser = argparse.ArgumentParser(description="Train Snake AI")
|
||||
parser.add_argument("--config", action="store_true",
|
||||
help="Show configuration UI before training")
|
||||
parser.add_argument("--model", type=str, default="models/snake_ai.zip",
|
||||
help="Path to save/load model")
|
||||
parser.add_argument("--no-cuda", action="store_true",
|
||||
help="Disable CUDA training")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get training configuration
|
||||
if args.config:
|
||||
from src.ai.config_ui import ConfigUI
|
||||
config_ui = ConfigUI()
|
||||
config = config_ui.run()
|
||||
if config is None: # User cancelled
|
||||
return
|
||||
else:
|
||||
# Load saved config or use defaults
|
||||
config_file = "training_config.json"
|
||||
if os.path.exists(config_file):
|
||||
with open(config_file, 'r') as f:
|
||||
config = json.load(f)
|
||||
else:
|
||||
config = {
|
||||
'timesteps': 2000000,
|
||||
'learning_rate': 0.0001,
|
||||
'batch_size': 512,
|
||||
'n_envs': 32,
|
||||
'n_steps': 512
|
||||
}
|
||||
|
||||
# Create visualization queue
|
||||
viz_queue = Queue()
|
||||
|
||||
# Start visualization in separate thread
|
||||
viz_thread = Thread(target=run_dashboard, args=(viz_queue,))
|
||||
viz_thread.start()
|
||||
|
||||
# Start training with configuration
|
||||
train_model(
|
||||
total_timesteps=config['timesteps'],
|
||||
viz_queue=viz_queue,
|
||||
model_path=args.model,
|
||||
cuda=not args.no_cuda,
|
||||
learning_rate=config['learning_rate'],
|
||||
batch_size=config['batch_size'],
|
||||
n_envs=config['n_envs'],
|
||||
n_steps=config['n_steps']
|
||||
)
|
||||
|
||||
# Wait for visualization to finish
|
||||
viz_thread.join()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,540 @@
|
||||
"""
|
||||
Training Visualization Dashboard
|
||||
|
||||
This module provides a real-time visualization of the training process,
|
||||
showing both the training metrics and evaluation games.
|
||||
"""
|
||||
|
||||
import pygame
|
||||
import numpy as np
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
import threading
|
||||
from queue import Queue, Empty
|
||||
import time
|
||||
|
||||
class TrainingDashboard:
|
||||
def __init__(self, width: int = 1920, height: int = 1080):
|
||||
"""Initialize the training dashboard."""
|
||||
self.width = width
|
||||
self.height = height
|
||||
|
||||
# Fonts
|
||||
self.title_font = None
|
||||
self.header_font = None
|
||||
self.text_font = None
|
||||
|
||||
# Colors
|
||||
self.colors = {
|
||||
'background': (0, 0, 0),
|
||||
'text': (200, 200, 200),
|
||||
'highlight': (0, 255, 0),
|
||||
'grid': (40, 40, 40),
|
||||
'border': (0, 255, 0),
|
||||
'snake': (0, 255, 0),
|
||||
'food': (255, 0, 0),
|
||||
'graph_bg': (20, 20, 20),
|
||||
'graph_line': (0, 255, 0),
|
||||
'graph_grid': (40, 40, 40)
|
||||
}
|
||||
|
||||
# Layout
|
||||
self.layout = {
|
||||
'margin': 20,
|
||||
'game_size': 400,
|
||||
'graph_height': 200,
|
||||
'metrics_width': 300
|
||||
}
|
||||
|
||||
# Training metrics history
|
||||
self.metrics_history = {
|
||||
'rewards': [],
|
||||
'scores': [],
|
||||
'lengths': []
|
||||
}
|
||||
self.max_history = 1000
|
||||
|
||||
# Current state
|
||||
self.current_state = None
|
||||
self.eval_state = None
|
||||
self.demo_state = None
|
||||
self.training_info = None
|
||||
self.model_info = None
|
||||
|
||||
# Performance tracking
|
||||
self.frame_times = []
|
||||
self.max_frame_times = 60
|
||||
|
||||
# Initialize the clock
|
||||
self.clock = None
|
||||
self.target_fps = 60
|
||||
|
||||
def run(self, update_queue: Queue) -> None:
|
||||
"""Run the dashboard, processing updates from the queue."""
|
||||
pygame.init()
|
||||
try:
|
||||
self.screen = pygame.display.set_mode((self.width, self.height))
|
||||
pygame.display.set_caption("Snake AI Training Dashboard")
|
||||
|
||||
# Initialize fonts after pygame is initialized
|
||||
self.title_font = pygame.font.Font(None, 48)
|
||||
self.header_font = pygame.font.Font(None, 36)
|
||||
self.text_font = pygame.font.Font(None, 24)
|
||||
|
||||
# Initialize clock
|
||||
self.clock = pygame.time.Clock()
|
||||
|
||||
running = True
|
||||
while running:
|
||||
# Handle events
|
||||
for event in pygame.event.get():
|
||||
if event.type == pygame.QUIT:
|
||||
running = False
|
||||
elif event.type == pygame.KEYDOWN:
|
||||
if event.key == pygame.K_ESCAPE:
|
||||
running = False
|
||||
|
||||
# Process updates from queue
|
||||
try:
|
||||
while True: # Process all available updates
|
||||
update_type, data = update_queue.get_nowait()
|
||||
if update_type == 'update':
|
||||
self.update(data)
|
||||
elif update_type == 'stop':
|
||||
running = False
|
||||
break
|
||||
except Empty:
|
||||
pass
|
||||
|
||||
# Render dashboard
|
||||
self.render()
|
||||
self.clock.tick(self.target_fps)
|
||||
finally:
|
||||
pygame.quit()
|
||||
|
||||
def update(self, data: Dict[str, Any]) -> None:
|
||||
"""Update the dashboard with new data."""
|
||||
self.current_state = data # Store entire data dict
|
||||
if 'training_info' in data:
|
||||
self.training_info = data['training_info']
|
||||
# Update metrics history
|
||||
for key in ['rewards', 'scores', 'lengths']:
|
||||
if key in self.training_info:
|
||||
self.metrics_history[key].append(self.training_info[key])
|
||||
if len(self.metrics_history[key]) > self.max_history:
|
||||
self.metrics_history[key].pop(0)
|
||||
|
||||
def draw_game_view(self, state: Dict[str, Any], position: Tuple[int, int], size: int, title: str) -> None:
|
||||
"""Draw a game view at the specified position."""
|
||||
if not state:
|
||||
return
|
||||
|
||||
x, y = position
|
||||
|
||||
# Draw game area background
|
||||
game_rect = pygame.Rect(x, y, size, size)
|
||||
pygame.draw.rect(self.screen, self.colors['grid'], game_rect)
|
||||
pygame.draw.rect(self.screen, self.colors['border'], game_rect, 2)
|
||||
|
||||
# Get time limit info directly from state
|
||||
steps_since_food = state.get('steps_since_food', 0)
|
||||
current_time_limit = state.get('current_time_limit', 100) # Default to base limit if not provided
|
||||
time_left = max(0, current_time_limit - steps_since_food)
|
||||
|
||||
# Calculate cell size based on grid dimensions
|
||||
cell_size = size // 30 # Assuming 30x30 grid
|
||||
|
||||
# Draw snake
|
||||
if "snake_body" in state:
|
||||
for segment in state["snake_body"]:
|
||||
segment_rect = pygame.Rect(
|
||||
x + segment[0] * cell_size,
|
||||
y + segment[1] * cell_size,
|
||||
cell_size, cell_size
|
||||
)
|
||||
pygame.draw.rect(self.screen, self.colors['snake'], segment_rect)
|
||||
|
||||
# Draw food
|
||||
if "food_position" in state:
|
||||
food_rect = pygame.Rect(
|
||||
x + state["food_position"][0] * cell_size,
|
||||
y + state["food_position"][1] * cell_size,
|
||||
cell_size, cell_size
|
||||
)
|
||||
pygame.draw.rect(self.screen, self.colors['food'], food_rect)
|
||||
|
||||
# Draw score and timer above game area
|
||||
score = state.get('score', 0)
|
||||
score_text = f"Score: {score}"
|
||||
timer_text = f"Time: {time_left}/{current_time_limit}" # Show both current and max time
|
||||
combined_text = f"{score_text} | {timer_text}"
|
||||
score_surface = self.text_font.render(combined_text, True, self.colors['text'])
|
||||
score_rect = score_surface.get_rect(midtop=(x + size//2, y - 25))
|
||||
self.screen.blit(score_surface, score_rect)
|
||||
|
||||
# Draw title below game area, including wrap mode for training view
|
||||
if title == "Training Episode" and 'wrap_around' in state:
|
||||
wrap_mode = "(Wrap)" if state['wrap_around'] else "(No Wrap)"
|
||||
title = f"{title} {wrap_mode}"
|
||||
title_text = title
|
||||
title_surface = self.header_font.render(title_text, True, self.colors['text'])
|
||||
title_rect = title_surface.get_rect(midtop=(x + size//2, y + size + 10))
|
||||
self.screen.blit(title_surface, title_rect)
|
||||
|
||||
def draw_metrics(self, x: int, y: int, width: int) -> None:
|
||||
"""Draw training metrics."""
|
||||
if not self.training_info:
|
||||
return
|
||||
|
||||
font = pygame.font.Font(None, 24)
|
||||
y_offset = 0
|
||||
|
||||
# Draw high scores
|
||||
high_score = self.training_info.get("high_score", 0)
|
||||
recent_high = self.training_info.get("recent_high_score", 0)
|
||||
|
||||
metrics = [
|
||||
("All-time High Score", high_score),
|
||||
("Recent High Score", recent_high),
|
||||
("", ""), # Empty line for spacing
|
||||
("Training Metrics", ""),
|
||||
("Total Steps", self.training_info.get('total_timesteps', 0)),
|
||||
("Episode Reward", f"{self.training_info.get('episode_reward', 0):.2f}"),
|
||||
("Mean Reward", f"{self.training_info.get('mean_reward', 0):.2f}"),
|
||||
("Episode Length", self.training_info.get('episode_length', 0)),
|
||||
("Mean Length", f"{self.training_info.get('mean_length', 0):.2f}"),
|
||||
("FPS", f"{len(self.frame_times) / sum(self.frame_times):.1f}" if self.frame_times else "0")
|
||||
]
|
||||
|
||||
for i, (label, value) in enumerate(metrics):
|
||||
text = f"{label}: {value}" if label else ""
|
||||
surface = self.text_font.render(text, True, self.colors['text'])
|
||||
self.screen.blit(surface, (x, y + i * 25))
|
||||
|
||||
def draw_graph(self, data: List[float], rect: pygame.Rect, title: str,
|
||||
color: Tuple[int, int, int], min_val: float, max_val: float,
|
||||
smoothing_window: int = 10) -> None:
|
||||
"""Draw a line graph of the given data.
|
||||
|
||||
Args:
|
||||
data: List of data points.
|
||||
rect: Rectangle area for drawing the graph.
|
||||
title: Title for the graph.
|
||||
color: Color for the graph line.
|
||||
min_val: Minimum value for scaling the y-axis.
|
||||
max_val: Maximum value for scaling the y-axis.
|
||||
smoothing_window: Optional smoothing window size for drawing
|
||||
long term trends. Increase this value for a smoother, longer-term trend.
|
||||
Default is 10.
|
||||
"""
|
||||
if not data:
|
||||
return
|
||||
|
||||
# Create a separate surface for the graph
|
||||
graph_surface = pygame.Surface((rect.width, rect.height))
|
||||
graph_surface.fill(self.colors['graph_bg'])
|
||||
|
||||
# Draw grid lines on the graph surface
|
||||
num_lines = 5
|
||||
for i in range(num_lines):
|
||||
y_line = i * rect.height / (num_lines - 1)
|
||||
pygame.draw.line(graph_surface, self.colors['graph_grid'],
|
||||
(0, y_line), (rect.width, y_line))
|
||||
|
||||
# Add padding to the min/max values for better visualization
|
||||
value_range = max_val - min_val
|
||||
if value_range == 0:
|
||||
value_range = 1.0
|
||||
padding = value_range * 0.1
|
||||
min_val_adjusted = min_val - padding
|
||||
max_val_adjusted = max_val + padding
|
||||
|
||||
# Smooth the data using the provided smoothing_window parameter
|
||||
window_size = smoothing_window if len(data) >= smoothing_window else len(data)
|
||||
if window_size > 1:
|
||||
smoothed_data = np.convolve(data, np.ones(window_size) / window_size, mode='valid')
|
||||
else:
|
||||
smoothed_data = data
|
||||
|
||||
points = []
|
||||
for i, value in enumerate(smoothed_data):
|
||||
x_point = i * rect.width / len(smoothed_data)
|
||||
y_point = rect.height - ((value - min_val_adjusted) * rect.height /
|
||||
(max_val_adjusted - min_val_adjusted))
|
||||
points.append((x_point, y_point))
|
||||
|
||||
if len(points) > 1:
|
||||
pygame.draw.lines(graph_surface, color, False, points, 2)
|
||||
|
||||
# Draw border on the graph surface
|
||||
pygame.draw.rect(graph_surface, self.colors['border'],
|
||||
pygame.Rect(0, 0, rect.width, rect.height), 1)
|
||||
|
||||
# Blit the graph surface onto the main screen
|
||||
self.screen.blit(graph_surface, rect)
|
||||
|
||||
# Draw title and current value
|
||||
current_value = data[-1] if data else 0
|
||||
title_text = f"{title} (Current: {current_value:.2f})"
|
||||
title_surface = self.text_font.render(title_text, True, self.colors['text'])
|
||||
title_rect = title_surface.get_rect(midtop=(rect.centerx, rect.top - 20))
|
||||
self.screen.blit(title_surface, title_rect)
|
||||
|
||||
# Draw value labels to the right of the graph
|
||||
label_spacing = rect.height / (num_lines - 1)
|
||||
label_x = rect.right + 10
|
||||
for i in range(num_lines):
|
||||
value = max_val_adjusted - (i * (max_val_adjusted - min_val_adjusted) / (num_lines - 1))
|
||||
label_text = f"{value:.1f}"
|
||||
label_surface = self.text_font.render(label_text, True, self.colors['text'])
|
||||
label_rect = label_surface.get_rect(
|
||||
left=label_x,
|
||||
centery=rect.top + i * label_spacing
|
||||
)
|
||||
self.screen.blit(label_surface, label_rect)
|
||||
|
||||
def draw_weights_visualization(self, x: int, y: int, width: int, height: int, weights_info: Dict) -> None:
|
||||
"""Draw the neural network weights visualization."""
|
||||
if not weights_info:
|
||||
return
|
||||
|
||||
# Get data
|
||||
action_probs = weights_info.get('action_probs', [0, 0, 0, 0])
|
||||
value_estimate = weights_info.get('value_estimate', 0.0)
|
||||
action_labels = weights_info.get('action_labels', ['Up', 'Right', 'Down', 'Left'])
|
||||
|
||||
# Draw background
|
||||
rect = pygame.Rect(x, y, width, height)
|
||||
pygame.draw.rect(self.screen, self.colors['graph_bg'], rect)
|
||||
pygame.draw.rect(self.screen, self.colors['border'], rect, 1)
|
||||
|
||||
# Draw title
|
||||
title = "Action Probabilities"
|
||||
title_surface = self.header_font.render(title, True, self.colors['text'])
|
||||
title_rect = title_surface.get_rect(midtop=(x + width//2, y + 5))
|
||||
self.screen.blit(title_surface, title_rect)
|
||||
|
||||
# Draw action probability bars
|
||||
bar_height = 20
|
||||
bar_spacing = 30
|
||||
bar_start_y = y + 50
|
||||
max_bar_width = width - 120 # Leave space for labels
|
||||
|
||||
for i, (prob, label) in enumerate(zip(action_probs, action_labels)):
|
||||
# Draw label
|
||||
label_surface = self.text_font.render(label, True, self.colors['text'])
|
||||
self.screen.blit(label_surface, (x + 10, bar_start_y + i * bar_spacing))
|
||||
|
||||
# Draw bar background
|
||||
bar_bg_rect = pygame.Rect(x + 70, bar_start_y + i * bar_spacing, max_bar_width, bar_height)
|
||||
pygame.draw.rect(self.screen, self.colors['grid'], bar_bg_rect)
|
||||
|
||||
# Draw probability bar
|
||||
bar_width = int(prob * max_bar_width)
|
||||
bar_rect = pygame.Rect(x + 70, bar_start_y + i * bar_spacing, bar_width, bar_height)
|
||||
pygame.draw.rect(self.screen, self.colors['highlight'], bar_rect)
|
||||
|
||||
# Draw probability value
|
||||
prob_text = f"{prob:.2f}"
|
||||
prob_surface = self.text_font.render(prob_text, True, self.colors['text'])
|
||||
self.screen.blit(prob_surface, (x + 80 + max_bar_width, bar_start_y + i * bar_spacing))
|
||||
|
||||
# Draw value estimate
|
||||
value_text = f"Value Estimate: {value_estimate:.2f}"
|
||||
value_surface = self.text_font.render(value_text, True, self.colors['text'])
|
||||
value_rect = value_surface.get_rect(midbottom=(x + width//2, y + height - 10))
|
||||
self.screen.blit(value_surface, value_rect)
|
||||
|
||||
def draw_progress_bar(self, x: int, y: int, width: int, height: int, progress: float, text: str) -> None:
|
||||
"""Draw a progress bar with text.
|
||||
|
||||
Args:
|
||||
x, y: Position of the progress bar
|
||||
width, height: Dimensions of the progress bar
|
||||
progress: Progress value between 0 and 1
|
||||
text: Text to display above the progress bar
|
||||
"""
|
||||
# Draw background
|
||||
bg_rect = pygame.Rect(x, y, width, height)
|
||||
pygame.draw.rect(self.screen, self.colors['graph_bg'], bg_rect)
|
||||
pygame.draw.rect(self.screen, self.colors['border'], bg_rect, 1)
|
||||
|
||||
# Draw progress
|
||||
if progress > 0:
|
||||
progress_width = int(width * progress)
|
||||
progress_rect = pygame.Rect(x, y, progress_width, height)
|
||||
pygame.draw.rect(self.screen, self.colors['highlight'], progress_rect)
|
||||
|
||||
# Draw text
|
||||
text_surface = self.text_font.render(text, True, self.colors['text'])
|
||||
text_rect = text_surface.get_rect(bottomleft=(x, y - 5))
|
||||
self.screen.blit(text_surface, text_rect)
|
||||
|
||||
def render(self) -> None:
|
||||
"""Render the dashboard."""
|
||||
if not self.current_state:
|
||||
return
|
||||
|
||||
self.screen.fill(self.colors['background'])
|
||||
|
||||
# Draw game views
|
||||
if 'training_state' in self.current_state:
|
||||
# Draw both training games
|
||||
self.draw_game_view(
|
||||
self.current_state['training_state'][0], # No wrap training
|
||||
(50, 50),
|
||||
250,
|
||||
"Training (No Wrap)"
|
||||
)
|
||||
self.draw_game_view(
|
||||
self.current_state['training_state'][1], # Wrap training
|
||||
(350, 50),
|
||||
250,
|
||||
"Training (Wrap)"
|
||||
)
|
||||
|
||||
if 'eval_states' in self.current_state:
|
||||
self.draw_game_view(
|
||||
self.current_state['eval_states'][0],
|
||||
(650, 50),
|
||||
250,
|
||||
"Evaluation (No Wrap)"
|
||||
)
|
||||
self.draw_game_view(
|
||||
self.current_state['eval_states'][1],
|
||||
(950, 50),
|
||||
250,
|
||||
"Evaluation (Wrap)"
|
||||
)
|
||||
|
||||
if 'demo_state' in self.current_state:
|
||||
self.draw_game_view(
|
||||
self.current_state['demo_state'],
|
||||
(1250, 50),
|
||||
250,
|
||||
"Demo Game"
|
||||
)
|
||||
|
||||
# Draw metrics
|
||||
if self.training_info:
|
||||
metrics_data = [
|
||||
(self.training_info['rewards_history'], "Episode Rewards"),
|
||||
(self.training_info['lengths_history'], "Episode Lengths"),
|
||||
(self.training_info['eval_scores_history'], "Evaluation Scores")
|
||||
]
|
||||
|
||||
for i, (data, label) in enumerate(metrics_data):
|
||||
if data: # Only draw if we have data
|
||||
rect = pygame.Rect(50, 400 + i * 200, 500, 150)
|
||||
self.draw_graph(
|
||||
data,
|
||||
rect,
|
||||
label,
|
||||
self.colors['graph_line'],
|
||||
min(data) if data else 0,
|
||||
max(data) if data else 1
|
||||
)
|
||||
|
||||
# Draw current metrics text
|
||||
metrics_text = [
|
||||
f"Total Steps: {self.training_info.get('total_timesteps', 0):,}",
|
||||
f"Mean Reward: {self.training_info.get('mean_reward', 0):.2f}",
|
||||
f"Mean Length: {self.training_info.get('mean_length', 0):.1f}",
|
||||
f"Mean Eval Score: {self.training_info.get('mean_eval_score', 0):.2f}",
|
||||
f"High Score: {self.training_info.get('high_score', 0)}",
|
||||
f"Recent High Score: {self.training_info.get('recent_high_score', 0)}"
|
||||
]
|
||||
|
||||
# Add vertical spacer between metrics sections
|
||||
metrics_text.insert(3, "") # Insert empty string as spacer after first 3 metrics
|
||||
|
||||
font = pygame.font.Font(None, 36)
|
||||
for i, text in enumerate(metrics_text):
|
||||
surface = font.render(text, True, self.colors['text'])
|
||||
self.screen.blit(surface, (600, 400 + i * 30))
|
||||
|
||||
# Draw action probabilities if available
|
||||
if 'weights_info' in self.current_state:
|
||||
self.draw_weights_visualization(
|
||||
1100, 400,
|
||||
300, 300,
|
||||
self.current_state['weights_info']
|
||||
)
|
||||
|
||||
# Draw trend indicator graph
|
||||
if self.training_info:
|
||||
# Calculate trend indicators
|
||||
window = 100 # Use last 100 episodes for trends
|
||||
rewards = self.training_info['rewards_history'][-window:]
|
||||
lengths = self.training_info['lengths_history'][-window:]
|
||||
eval_scores = self.training_info['eval_scores_history'][-window:]
|
||||
|
||||
if rewards and lengths and eval_scores:
|
||||
# Normalize each metric to 0-1 range for fair combination
|
||||
norm_rewards = [(r - min(rewards)) / (max(rewards) - min(rewards) + 1e-8) for r in rewards]
|
||||
norm_lengths = [(l - min(lengths)) / (max(lengths) - min(lengths) + 1e-8) for l in lengths]
|
||||
norm_scores = [(s - min(eval_scores)) / (max(eval_scores) - min(eval_scores) + 1e-8) for s in eval_scores]
|
||||
|
||||
# Combine metrics with weights (rewards: 0.4, lengths: 0.3, eval_scores: 0.3)
|
||||
trend_data = [0.4 * r + 0.3 * l + 0.3 * s
|
||||
for r, l, s in zip(norm_rewards, norm_lengths, norm_scores)]
|
||||
# Add trend direction indicator
|
||||
|
||||
trending = 0 # -1, 0, 1
|
||||
|
||||
if len(trend_data) >= 2:
|
||||
recent_trend = trend_data[-1] - trend_data[0]
|
||||
trending = 1 if recent_trend > 0.1 else -1 if recent_trend < -0.1 else 0
|
||||
trend_text = "↑ Improving" if trending == 1 else "↓ Declining" if trending == -1 else "→ Stable"
|
||||
trend_color = (0, 255, 0) if trending == 1 else (255, 0, 0) if trending == -1 else (200, 200, 0)
|
||||
trend_surface = self.text_font.render(trend_text, True, trend_color)
|
||||
trend_rect = trend_surface.get_rect(midtop=(1250, 860))
|
||||
self.screen.blit(trend_surface, trend_rect)
|
||||
|
||||
graph_color = (0, 255, 0) if trending == 1 else (255, 0, 0) if trending == -1 else (200, 200, 0)
|
||||
|
||||
# Draw trend graph 50 pixels below the action probabilities graph
|
||||
trend_rect = pygame.Rect(1100, 750, 300, 150)
|
||||
self.draw_graph(
|
||||
trend_data,
|
||||
trend_rect,
|
||||
"Learning Trend",
|
||||
graph_color, # color based on trend
|
||||
0, # Min value (normalized)
|
||||
1, # Max value (normalized)
|
||||
smoothing_window=10 # Use a larger smoothing window for longer-term trends
|
||||
)
|
||||
|
||||
# Draw training progress bar at the bottom of the screen
|
||||
if self.training_info:
|
||||
# Get current session steps (from start of this training run)
|
||||
current_session_steps = self.training_info.get('total_timesteps', 0) - self.training_info.get('initial_timesteps', 0)
|
||||
target_timesteps = self.training_info.get('target_timesteps', 1000000)
|
||||
progress = min(1.0, current_session_steps / target_timesteps)
|
||||
|
||||
# Show both current session progress and total model training
|
||||
total_model_steps = self.training_info.get('total_timesteps', 0)
|
||||
progress_text = (
|
||||
f"Current Session: {current_session_steps:,}/{target_timesteps:,} steps ({progress*100:.1f}%) | "
|
||||
f"Total Model Training: {total_model_steps:,} steps"
|
||||
)
|
||||
self.draw_progress_bar(50, self.height - 40, self.width - 100, 20, progress, progress_text)
|
||||
|
||||
pygame.display.flip()
|
||||
|
||||
def handle_events(self):
|
||||
for event in pygame.event.get():
|
||||
if event.type == pygame.QUIT:
|
||||
self.running = False
|
||||
elif event.type == pygame.KEYDOWN:
|
||||
if event.key == pygame.K_ESCAPE:
|
||||
self.running = False
|
||||
|
||||
def run_dashboard(update_queue: Queue) -> None:
|
||||
"""Run the training dashboard in a separate thread."""
|
||||
dashboard = TrainingDashboard()
|
||||
try:
|
||||
dashboard.run(update_queue)
|
||||
finally:
|
||||
# Send stop signal back to training process
|
||||
update_queue.put(('stop_training', None))
|
||||
pygame.quit()
|
||||
Reference in new issue
Block a user