Major code restructuring: Reorganized project structure, added AI components - Added new module structure (ai, config, core, ui) - Moved snake and food logic into core module - Added training configuration - Updated gitignore for project-specific files - Modified tests to match new structure

This commit is contained in:
Rbanh committed 2025-02-24 18:38:44 -05:00
1 parent 251822ec35
commit b8eeaa4739
34 files changed
+3928 -513

No files matched your search

+235
View File
@@ -0,0 +1,235 @@
"""
Training Configuration UI
This module provides a graphical interface for configuring AI training parameters.
"""
import pygame
import json
import os
from typing import Dict, Any, Optional, Tuple, List
class ConfigUI:
def __init__(self, width: int = 800, height: int = 600):
"""Initialize the configuration UI."""
pygame.init()
self.width = width
self.height = height
self.screen = pygame.display.set_mode((width, height))
pygame.display.set_caption("Training Configuration")
# Fonts
self.title_font = pygame.font.Font(None, 48)
self.header_font = pygame.font.Font(None, 36)
self.text_font = pygame.font.Font(None, 24)
# Colors
self.colors = {
'background': (0, 0, 0),
'text': (200, 200, 200),
'highlight': (0, 255, 0),
'button': (40, 40, 40),
'button_hover': (60, 60, 60),
'input_bg': (30, 30, 30),
'input_active': (50, 50, 50)
}
# Parameters
self.parameters = {
'timesteps': {
'value': 1000000,
'type': 'int',
'min': 1000,
'max': 10000000,
'description': 'Total timesteps to train for'
},
'learning_rate': {
'value': 0.0003,
'type': 'float',
'min': 0.00001,
'max': 0.01,
'description': 'Learning rate for training'
},
'batch_size': {
'value': 64,
'type': 'int',
'min': 32,
'max': 512,
'description': 'Batch size for training'
},
'n_envs': {
'value': 8,
'type': 'int',
'min': 1,
'max': 32,
'description': 'Number of parallel environments'
},
'n_steps': {
'value': 2048,
'type': 'int',
'min': 128,
'max': 8192,
'description': 'Number of steps per update'
}
}
# UI state
self.active_input = None
self.input_text = ""
self.scroll_offset = 0
self.max_scroll = max(0, len(self.parameters) * 60 - (height - 200))
# Load saved config if exists
self.config_file = "training_config.json"
self.load_config()
def load_config(self) -> None:
"""Load configuration from file."""
if os.path.exists(self.config_file):
try:
with open(self.config_file, 'r') as f:
saved_config = json.load(f)
for key, value in saved_config.items():
if key in self.parameters:
self.parameters[key]['value'] = value
except:
pass
def save_config(self) -> None:
"""Save configuration to file."""
config = {key: param['value'] for key, param in self.parameters.items()}
with open(self.config_file, 'w') as f:
json.dump(config, f, indent=4)
def draw_text_input(self, rect: pygame.Rect, value: Any, active: bool) -> None:
"""Draw a text input field."""
color = self.colors['input_active'] if active else self.colors['input_bg']
pygame.draw.rect(self.screen, color, rect)
pygame.draw.rect(self.screen, self.colors['text'], rect, 1)
text = str(value)
if active:
text = self.input_text + "|"
text_surface = self.text_font.render(text, True, self.colors['text'])
text_rect = text_surface.get_rect(midleft=(rect.left + 5, rect.centery))
self.screen.blit(text_surface, text_rect)
def draw_button(self, rect: pygame.Rect, text: str, hover: bool = False) -> None:
"""Draw a button."""
color = self.colors['button_hover'] if hover else self.colors['button']
pygame.draw.rect(self.screen, color, rect)
pygame.draw.rect(self.screen, self.colors['text'], rect, 1)
text_surface = self.text_font.render(text, True, self.colors['text'])
text_rect = text_surface.get_rect(center=rect.center)
self.screen.blit(text_surface, text_rect)
def validate_input(self, param: Dict[str, Any], value: str) -> Optional[Any]:
"""Validate and convert input value."""
try:
if param['type'] == 'int':
val = int(value)
else:
val = float(value)
if val < param['min'] or val > param['max']:
return None
return val
except:
return None
def run(self) -> Optional[Dict[str, Any]]:
"""Run the configuration UI. Returns the config dict if saved, None if cancelled."""
running = True
save_clicked = False
mouse_pos = (0, 0)
while running:
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
elif event.type == pygame.MOUSEBUTTONDOWN:
mouse_pos = event.pos
# Check parameter inputs
y = 100 - self.scroll_offset
for name, param in self.parameters.items():
input_rect = pygame.Rect(300, y, 200, 30)
if input_rect.collidepoint(mouse_pos):
self.active_input = name
self.input_text = str(param['value'])
y += 60
# Check buttons
save_rect = pygame.Rect(self.width//2 - 150, self.height - 60, 140, 40)
cancel_rect = pygame.Rect(self.width//2 + 10, self.height - 60, 140, 40)
if save_rect.collidepoint(mouse_pos):
self.save_config()
save_clicked = True
running = False
elif cancel_rect.collidepoint(mouse_pos):
running = False
elif event.type == pygame.MOUSEBUTTONUP:
if event.button == 4: # Mouse wheel up
self.scroll_offset = max(0, self.scroll_offset - 30)
elif event.button == 5: # Mouse wheel down
self.scroll_offset = min(self.max_scroll, self.scroll_offset + 30)
elif event.type == pygame.KEYDOWN:
if self.active_input is not None:
if event.key == pygame.K_RETURN:
param = self.parameters[self.active_input]
if val := self.validate_input(param, self.input_text):
param['value'] = val
self.active_input = None
elif event.key == pygame.K_BACKSPACE:
self.input_text = self.input_text[:-1]
else:
if event.unicode.isnumeric() or event.unicode == '.':
self.input_text += event.unicode
# Draw UI
self.screen.fill(self.colors['background'])
# Title
title = "Training Configuration"
title_surface = self.title_font.render(title, True, self.colors['highlight'])
title_rect = title_surface.get_rect(midtop=(self.width//2, 20))
self.screen.blit(title_surface, title_rect)
# Parameters
y = 100 - self.scroll_offset
for name, param in self.parameters.items():
if 0 <= y <= self.height - 100:
# Parameter name and description
name_surface = self.header_font.render(name, True, self.colors['text'])
desc_surface = self.text_font.render(param['description'], True, self.colors['text'])
self.screen.blit(name_surface, (20, y))
self.screen.blit(desc_surface, (20, y + 30))
# Input field
input_rect = pygame.Rect(300, y, 200, 30)
self.draw_text_input(input_rect, param['value'], name == self.active_input)
y += 60
# Buttons
save_rect = pygame.Rect(self.width//2 - 150, self.height - 60, 140, 40)
cancel_rect = pygame.Rect(self.width//2 + 10, self.height - 60, 140, 40)
save_hover = save_rect.collidepoint(mouse_pos)
cancel_hover = cancel_rect.collidepoint(mouse_pos)
self.draw_button(save_rect, "Save", save_hover)
self.draw_button(cancel_rect, "Cancel", cancel_hover)
pygame.display.flip()
pygame.quit()
if save_clicked:
return {key: param['value'] for key, param in self.parameters.items()}
return None
+70
View File
@@ -0,0 +1,70 @@
"""
AI Controller for Snake Game
This module provides the AI controller that uses trained models to play the game.
It handles model loading, state processing, and decision making during gameplay.
"""
import os
import numpy as np
from stable_baselines3 import PPO
from ai.environment import SnakeEnv, Direction
class AIController:
"""AI controller that uses trained models to play the game."""
def __init__(self, difficulty: str = "medium"):
"""
Initialize the AI controller.
Args:
difficulty: "easy", "medium", or "hard"
"""
self.difficulty = difficulty
self.model = None
self.env = SnakeEnv() # For state processing
self._load_model()
def _load_model(self):
"""Load the appropriate model based on difficulty."""
model_path = f"models/{self.difficulty}/best_model.zip"
if not os.path.exists(model_path):
# Fall back to final model if best model doesn't exist
model_path = f"models/{self.difficulty}/final_model.zip"
if not os.path.exists(model_path):
raise FileNotFoundError(
f"No model found for difficulty {self.difficulty}. "
"Please train the model first."
)
self.model = PPO.load(model_path)
def get_action(self, game_state: dict) -> Direction:
"""
Get the next action based on the current game state.
Args:
game_state: Dictionary containing:
- snake: Snake object
- food: Food object
- width: Game width
- height: Game height
Returns:
Direction enum indicating the chosen action
"""
# Update environment with current game state
self.env.snake = game_state["snake"]
self.env.food = game_state["food"]
self.env.width = game_state["width"]
self.env.height = game_state["height"]
# Get state observation
state = self.env._get_state()
# Get action from model
action, _ = self.model.predict(state, deterministic=True)
# Convert action index to Direction
return self.env.action_space[action]
+787
View File
@@ -0,0 +1,787 @@
"""
Snake Game Environment for Reinforcement Learning
This module provides a gym-like interface for training AI agents to play the snake game.
It includes:
- State observation space
- Action space
- Reward system
- Environment dynamics
"""
import numpy as np
import random
import gymnasium as gym
from gymnasium import spaces
from typing import Tuple, List, Dict, Any, Optional
import pygame
from src.core import Snake
from src.core import Direction
from src.core import GameSession
from src.ui import GameArea
from src.config import GameRules
def manhattan_distance(pos1: Tuple[int, int], pos2: Tuple[int, int]) -> int:
"""Calculate Manhattan distance between two points."""
return abs(pos1[0] - pos2[0]) + abs(pos1[1] - pos2[1])
def manhattan_distance_wrap(pos1: Tuple[int, int], pos2: Tuple[int, int], size: int) -> int:
"""Calculate Manhattan distance between two points with wrap-around."""
dx = abs(pos1[0] - pos2[0])
dy = abs(pos1[1] - pos2[1])
return min(dx, size - dx) + min(dy, size - dy)
class SnakeEnv(gym.Env):
"""A Gymnasium environment for training snake AI agents."""
metadata = {"render_modes": ["human", "rgb_array"], "render_fps": 30}
def __init__(self, size=30, render_mode=None, difficulty="easy"):
super().__init__()
self.size = size
self.render_mode = render_mode
self.difficulty = difficulty
# Action to Direction mapping
self.action_to_direction = {
0: Direction.UP,
1: Direction.RIGHT,
2: Direction.DOWN,
3: Direction.LEFT
}
# Direction to index mapping for one-hot encoding
self.direction_to_index = {
Direction.UP: 0,
Direction.RIGHT: 1,
Direction.DOWN: 2,
Direction.LEFT: 3
}
# Initialize reward scales with emphasis on precision
self.reward_scales = {
'food': 20.0, # Base food reward (will be scaled exponentially)
'death': -5.0, # Reduced death penalty to encourage exploration
'distance': 0.5, # Increased distance reward
'survival': -0.01, # Small survival penalty
'milestone': 10.0, # Larger milestone rewards
'efficiency': 0.2, # Small efficiency reward
'exploration': 0.5, # Increased exploration reward
'safety': 1.0, # Safety reward for avoiding danger
'timeout': -1.0, # Reduced timeout penalty
'wrap_bonus': 0.2, # Small wrap-around bonus
'near_miss': -0.1, # Very small near miss penalty
'repetitive': -2 # Increased base penalty for repetitive movement
}
# Near miss detection
self.near_miss_threshold = 1 # Only penalize very close misses
# Time limit parameters - more lenient
self.base_time_limit = 200 # Double base time limit
self.length_time_bonus = 100 # Double length bonus
self.max_time_limit = 1000 # Much longer maximum time
# Exploration decay
self.initial_exploration_bonus = 2.0 # Higher initial exploration
self.exploration_decay = 0.9995 # Slower decay
self.min_exploration_bonus = 0.5 # Higher minimum exploration
self.current_exploration_bonus = self.initial_exploration_bonus
self.total_episodes = 0 # Track number of episodes for decay
# Curriculum learning parameters
self.curriculum_stage = 0
self.success_threshold = {
0: 3, # Stage 0: Basic movement (need more consistent success)
1: 5, # Stage 1: Food collection
2: 7, # Stage 2: Longer snake
3: 10 # Stage 3: Full difficulty
}
self.consecutive_successes = 0
self.stage_requirements = {
0: 1, # Stage 0: Get one food
1: 3, # Stage 1: Get three food
2: 5, # Stage 2: Get five food
3: 7 # Stage 3: Get seven food
}
# Dynamic episode length based on curriculum stage
self.base_max_steps = 300 # More steps for exploration
self.max_steps = self.base_max_steps
# Window dimensions for rendering
self.window_width = 1024
self.window_height = 768
# Create game area and rules
self.game_area = GameArea(self.window_width, self.window_height)
self.rules = GameRules()
self.rules.wrap_around = False # No wrap-around for AI training
# Define action and observation spaces
self.action_space = spaces.Discrete(4) # Up, Right, Down, Left
num_channels = 26 # Total number of channels we created in _get_normalized_observation
self.observation_space = spaces.Box(
low=0,
high=1,
shape=(num_channels, size, size), # (channels, height, width)
dtype=np.float32
)
# Game session
self.session = None
self.steps = 0
self.steps_since_food = 0
self.current_time = 0
self.last_score = 0
self.last_distance = float('inf')
self.last_food_steps = 0
self.steps_since_direction_change = 0 # Track steps since last direction change
self.last_direction = None # Track previous direction
self.recent_positions = []
self.steps_in_same_direction = 0 # Track steps without direction change
self.last_min_food_distance = float('inf') # Initialize minimum distance tracking
# For rendering
self.window = None
self.clock = None
self.score_milestone_rewards = {5: 10.0, 10: 20.0, 15: 30.0} # Bonus rewards at score milestones
self.last_milestone = 0 # Track last milestone reached
self.base_survival_penalty = -1.0 # Much larger constant penalty per step
# Movement control parameters
self.max_direction_steps = 10 # Reduced from 20 to 10
self.direction_step_increase = 2 # Reduced from 5 to 2
self.min_safe_turns = 2 # Reduced from 3 to 2
self.repetitive_threshold = 10 # Base threshold before repetitive penalty kicks in
self.repetitive_scale = 1.2 # Base scale for penalty growth
self.repetitive_wrap_scale = 2.0 # Much faster growth rate in wrap mode
self.max_repetitive_penalty = -5.0 # Increased maximum repetitive movement penalty
# Initialize base rewards with adjusted values
rewards = {
'food': 0.0,
'death': 0.0,
'distance': 0.0,
'survival': self.base_survival_penalty * 0.5,
'direction': 0.0,
'efficiency': 0.0,
'progress': 0.0,
'alignment': 0.0,
'repetitive': 0.0
}
def _get_normalized_observation(self) -> np.ndarray:
"""
Create a multi-channel 2D observation from the game state.
Spatial channels encode the grid layout (snake & food),
while extra scalar features are broadcast over additional channels.
"""
state = self.session.get_state()
snake_body = state["snake_body"]
food_position = state["food_position"]
head = snake_body[0]
body = snake_body
# --- Spatial Channels ---
# Channel 0: Snake positions (1 for snake, 0 otherwise)
snake_channel = np.zeros((self.size, self.size), dtype=np.float32)
for pos in snake_body:
snake_channel[pos[1], pos[0]] = 1.0
assert pos[0] < self.size and pos[1] < self.size, \
f"Snake position {pos} exceeds grid size {self.size}"
# Channel 1: Food position
food_channel = np.zeros((self.size, self.size), dtype=np.float32)
food_channel[food_position[1], food_position[0]] = 1.0
# --- Auxiliary Scalar Features ---
# Compute distances and apply wrap-around adjustments
dx = food_position[0] - head[0]
dy = food_position[1] - head[1]
if self.rules.wrap_around:
if dx > self.size/2:
wrap_dx = dx - self.size
elif dx < -self.size/2:
wrap_dx = dx + self.size
else:
wrap_dx = dx
if dy > self.size/2:
wrap_dy = dy - self.size
elif dy < -self.size/2:
wrap_dy = dy + self.size
else:
wrap_dy = dy
else:
wrap_dx = dx
wrap_dy = dy
# Normalize
dx_norm = dx / self.size
dy_norm = dy / self.size
wrap_dx_norm = wrap_dx / self.size
wrap_dy_norm = wrap_dy / self.size
# Danger indicators (vector of 4 values)
dangers = self._get_danger_observations_all_directions(head, body)
# Other auxiliary scalar features
snake_length_norm = len(body) / (self.size**2)
# One-hot encode current direction
direction_one_hot = [0.0] * 4
direction_one_hot[self.direction_to_index[state["snake_direction"]]] = 1.0
# One-hot encode relative food direction
food_dir = [0.0] * 4
if abs(dx_norm) > abs(dy_norm):
if dx_norm > 0:
food_dir[self.direction_to_index[Direction.RIGHT]] = 1.0
else:
food_dir[self.direction_to_index[Direction.LEFT]] = 1.0
else:
if dy_norm > 0:
food_dir[self.direction_to_index[Direction.DOWN]] = 1.0
else:
food_dir[self.direction_to_index[Direction.UP]] = 1.0
current_time_limit = min(
self.max_time_limit,
self.base_time_limit + (len(body) - 1) * self.length_time_bonus
)
time_pressure = np.clip(self.steps_since_food / current_time_limit, 0, 1)
wrap_mode = float(self.rules.wrap_around)
progress = min(1.0, len(body) / 10)
recent_food = min(1.0, self.steps_since_food / 50)
curriculum_stage_norm = float(self.curriculum_stage) / 3
advanced_length = float(len(body) > 5)
manhattan_norm = min(1.0, manhattan_distance(head, food_position) / (self.size * 2))
# --- Create "broadcasted" channels for each scalar ---
def broadcast_channel(value):
return np.full((self.size, self.size), value, dtype=np.float32)
dx_channel = broadcast_channel(dx_norm)
dy_channel = broadcast_channel(dy_norm)
wrap_dx_channel = broadcast_channel(wrap_dx_norm)
wrap_dy_channel = broadcast_channel(wrap_dy_norm)
snake_length_chan = broadcast_channel(snake_length_norm)
time_pressure_chan = broadcast_channel(time_pressure)
wrap_mode_chan = broadcast_channel(wrap_mode)
progress_chan = broadcast_channel(progress)
recent_food_chan = broadcast_channel(recent_food)
curriculum_chan = broadcast_channel(curriculum_stage_norm)
advanced_length_chan = broadcast_channel(advanced_length)
manhattan_chan = broadcast_channel(manhattan_norm)
# For one-hot features, create one channel per value
direction_channels = [broadcast_channel(v) for v in direction_one_hot]
food_dir_channels = [broadcast_channel(v) for v in food_dir]
# For dangers, one channel per direction (4 channels)
danger_channels = [broadcast_channel(d) for d in dangers]
# --- Stack all channels ---
# You can adjust the channel order as desired.
channels = [
snake_channel, # Channel 0
food_channel, # Channel 1
dx_channel, # Channel 2
dy_channel, # Channel 3
wrap_dx_channel, # Channel 4
wrap_dy_channel, # Channel 5
]
channels.extend(danger_channels) # Channels 6-9
channels.append(snake_length_chan) # Channel 10
channels.extend(direction_channels) # Channels 11-14
channels.extend(food_dir_channels) # Channels 15-18
channels.append(time_pressure_chan) # Channel 19
channels.append(wrap_mode_chan) # Channel 20
channels.append(progress_chan) # Channel 21
channels.append(recent_food_chan) # Channel 22
channels.append(curriculum_chan) # Channel 23
channels.append(advanced_length_chan) # Channel 24
channels.append(manhattan_chan) # Channel 25
# Final observation: shape (num_channels, size, size)
observation = np.stack(channels, axis=0)
return np.clip(observation, 0, 1)
def _get_danger_observations_all_directions(self, head: Tuple[int, int], body: List[Tuple[int, int]]) -> List[float]:
"""Get danger observations in all four directions."""
dangers = [0.0] * 4 # [UP, RIGHT, DOWN, LEFT]
# Check each direction
directions = [Direction.UP, Direction.RIGHT, Direction.DOWN, Direction.LEFT]
for i, direction in enumerate(directions):
dx, dy = direction.to_vector()
next_pos = (head[0] + dx, head[1] + dy)
# Check wall collision
if not self.rules.wrap_around:
if (next_pos[0] < 0 or next_pos[0] >= self.size or
next_pos[1] < 0 or next_pos[1] >= self.size):
dangers[i] = 1.0
continue
else:
next_pos = (next_pos[0] % self.size, next_pos[1] % self.size)
# Check self collision
if next_pos in body[1:]:
dangers[i] = 1.0
return dangers
def reset(self, seed=None, options=None):
"""Reset with curriculum-based difficulty and exploration decay."""
# Call parent reset without unpacking
super().reset(seed=seed)
if seed is not None:
random.seed(seed)
np.random.seed(seed)
# Update exploration bonus
self.total_episodes += 1
self.current_exploration_bonus = max(
self.min_exploration_bonus,
self.initial_exploration_bonus * (self.exploration_decay ** self.total_episodes)
)
# Initialize game session with curriculum-appropriate settings
self.session = GameSession(
self.size,
self.size,
self.rules
)
# Reset counters
self.steps = 0
self.steps_since_food = 0
self.current_time = 0
self.last_score = 0
self.last_distance = float('inf')
self.last_direction = self.session.get_state()["snake_direction"] # Track initial direction
self.recent_positions = []
self.steps_in_same_direction = 0 # Track steps without direction change
self.last_min_food_distance = float('inf') # Initialize minimum distance tracking
# Get initial observation
observation = self._get_normalized_observation()
info = {
"curriculum_stage": self.curriculum_stage,
"exploration_bonus": self.current_exploration_bonus
}
return observation, info
def _calculate_area_penalty(self, head_pos):
"""Calculate penalty for staying in a small area."""
# Add current position to history
self.recent_positions.append(head_pos)
if len(self.recent_positions) > self.max_recent_positions:
self.recent_positions.pop(0)
if len(self.recent_positions) < 10: # Need minimum history to calculate
return 0.0
# Calculate bounding box of recent positions
x_coords = [p[0] for p in self.recent_positions]
y_coords = [p[1] for p in self.recent_positions]
area_width = max(x_coords) - min(x_coords) + 1
area_height = max(y_coords) - min(y_coords) + 1
area = area_width * area_height
# Calculate unique positions visited
unique_positions = len(set(self.recent_positions))
# Penalize small areas and repeated positions
area_penalty = 0.0
if area < 9: # 3x3 grid or smaller
area_penalty = -0.2 * (9 - area) / 8 # Max penalty -0.2 for 1x1 area
# Additional penalty for revisiting same positions frequently
repetition_penalty = -0.3 * (1 - unique_positions / len(self.recent_positions))
return area_penalty + repetition_penalty
def step(self, action):
"""Improved reward structure with length-aware exploration."""
self.steps += 1
self.steps_since_food += 1
# Calculate current time limit based on snake length
snake_length = len(self.session.get_state()["snake_body"])
current_time_limit = min(
self.max_time_limit,
self.base_time_limit + (snake_length - 1) * self.length_time_bonus
)
# Get state before action
prev_state = self.session.get_state()
prev_head = prev_state["snake_head"]
prev_food_dist = manhattan_distance(prev_head, prev_state["food_position"])
if self.rules.wrap_around:
prev_food_dist = min(prev_food_dist,
manhattan_distance_wrap(prev_head, prev_state["food_position"], self.size))
# Convert action to Direction and apply
direction = self.action_to_direction[int(action)]
# Update game state
state, base_reward, done = self.session.step(direction, self.current_time)
# Initialize reward components
rewards = {
'food': 0.0,
'death': 0.0,
'distance': 0.0,
'survival': 0.0,
'milestone': 0.0,
'efficiency': 0.0,
'exploration': 0.0,
'safety': 0.0,
'timeout': 0.0,
'direction': 0.0,
'repetitive': 0.0
}
# Get current state info
curr_head = state["snake_body"][0]
curr_food = state["food_position"]
curr_food_dist = manhattan_distance(curr_head, curr_food)
if self.rules.wrap_around:
curr_food_dist = min(curr_food_dist,
manhattan_distance_wrap(curr_head, curr_food, self.size))
# Update current time
self.current_time += self.session.move_cooldown
# Food reward with exponential scaling based on snake length
food_eaten = state["score"] > prev_state["score"]
if food_eaten:
# Reset timer when food is eaten
self.steps_since_food = 0
# Calculate exponential food reward
base_food_reward = self.reward_scales['food']
length_multiplier = 1.2 ** (len(state["snake_body"]) - 1) # 20% increase per length
rewards['food'] = base_food_reward * length_multiplier
# Add milestone rewards for achieving certain scores
if state["score"] in [5, 10, 15, 20]:
rewards['milestone'] = self.reward_scales['milestone'] * (state["score"] / 5)
else:
# Only apply survival penalty if we haven't eaten food in a while
if self.steps_since_food > current_time_limit / 2:
penalty_factor = (self.steps_since_food - current_time_limit/2) / (current_time_limit/2)
rewards['survival'] = self.reward_scales['survival'] * penalty_factor
# Check if direction changed and update counters
direction_changed = direction != self.last_direction
if direction_changed:
self.steps_in_same_direction = 0
else:
self.steps_in_same_direction += 1
# Apply growing penalty for repetitive movement
if self.steps_in_same_direction > (self.repetitive_threshold - 5 if self.rules.wrap_around else self.repetitive_threshold):
excess_steps = self.steps_in_same_direction - (self.repetitive_threshold - 5 if self.rules.wrap_around else self.repetitive_threshold)
# Use much faster growth rate in wrap-around mode
growth_rate = self.repetitive_wrap_scale if self.rules.wrap_around else self.repetitive_scale
repetitive_penalty = self.reward_scales['repetitive'] * (growth_rate ** excess_steps)
# Apply additional multiplier in wrap-around mode that grows with steps
if self.rules.wrap_around:
wrap_multiplier = 1.0 + (excess_steps * 0.5) # Multiplier grows with each step
repetitive_penalty *= wrap_multiplier
rewards['repetitive'] = repetitive_penalty # No maximum cap - let it grow unbounded in wrap-around mode
else:
# Cap the penalty when not in wrap-around mode
rewards['repetitive'] = max(repetitive_penalty, self.max_repetitive_penalty)
self.last_direction = direction
# Check for timeout or stuck in loop - only if we haven't just eaten food
if (self.steps_since_food >= current_time_limit and not food_eaten):
done = True
rewards['timeout'] = self.reward_scales['timeout']
# Scale timeout penalty based on distance to food
if curr_food_dist < 5: # Harsher penalty for timing out near food
rewards['timeout'] *= 1.5
# Additional penalty for getting stuck in a loop
if self.steps_in_same_direction >= 30: # Also update this threshold
rewards['timeout'] *= 1.2
# Death penalty - calculate AFTER food rewards
if done and not self.steps_since_food >= current_time_limit:
base_death_penalty = self.reward_scales['death']
# Check if death was due to self-collision
if curr_head in state["snake_body"][1:]:
# Base multiplier for self-collision
self_collision_multiplier = 2.0
# Additional penalty scaling with score
# At score 5: 2.5x penalty
# At score 10: 3.0x penalty
# At score 15: 3.5x penalty
score_multiplier = 1.0 + (state["score"] / 10)
# Apply both multipliers
rewards['death'] = base_death_penalty * self_collision_multiplier * score_multiplier
# Additional penalty if we died near food
if curr_food_dist < 5:
rewards['death'] *= 1.5 # Even harsher if we collide with ourselves near food
else:
# For wall collisions, keep original scaling but make it less punishing at higher scores
rewards['death'] = base_death_penalty * (1.0 - min(0.5, state["score"] / 20))
# Add an immediate negative reward for losing potential score
potential_loss_penalty = -0.5 * state["score"] # Larger penalty for dying with higher scores
rewards['death'] += potential_loss_penalty
# Track minimum distance to food and check for near misses
if curr_food_dist < self.last_min_food_distance:
self.last_min_food_distance = curr_food_dist
elif curr_food_dist > self.last_min_food_distance:
# We're moving away from our closest approach
if self.last_min_food_distance <= self.near_miss_threshold:
# We got very close but missed
rewards['distance'] += self.reward_scales['near_miss']
self.last_min_food_distance = float('inf') # Reset tracking
# Progressive distance reward - more reward for getting closer when near food
distance_change = prev_food_dist - curr_food_dist
if curr_food_dist < 5: # When close to food
distance_multiplier = 2.0 # Double the reward/penalty
else:
distance_multiplier = 1.0
rewards['distance'] = distance_change * self.reward_scales['distance'] * distance_multiplier
# Calculate direction penalty
if curr_food_dist <= 5: # Only apply when close to food
# Calculate optimal direction to food
dx = curr_food[0] - curr_head[0]
dy = curr_food[1] - curr_head[1]
if self.rules.wrap_around:
# Adjust for wrap-around
if dx > self.size/2: dx -= self.size
elif dx < -self.size/2: dx += self.size
if dy > self.size/2: dy -= self.size
elif dy < -self.size/2: dy += self.size
# Determine optimal direction(s)
optimal_directions = []
if abs(dx) > abs(dy):
if dx > 0: optimal_directions.append(Direction.RIGHT)
elif dx < 0: optimal_directions.append(Direction.LEFT)
if abs(dy) >= abs(dx):
if dy > 0: optimal_directions.append(Direction.DOWN)
elif dy < 0: optimal_directions.append(Direction.UP)
# Base penalty scales with proximity (closer = higher penalty)
base_penalty = (6 - curr_food_dist) * 0.1 # Scales from 0.1 to 0.5
if direction not in optimal_directions:
# Check if we're moving directly away from food
opposite_dirs = {
Direction.UP: Direction.DOWN,
Direction.DOWN: Direction.UP,
Direction.LEFT: Direction.RIGHT,
Direction.RIGHT: Direction.LEFT
}
optimal_opposites = [opposite_dirs[d] for d in optimal_directions]
if direction in optimal_opposites:
# Double penalty for moving directly away from food
rewards['direction'] = -2 * base_penalty
else:
# Regular penalty for suboptimal direction
rewards['direction'] = -base_penalty
elif curr_food_dist <= 2: # Small reward for correct direction when very close
rewards['direction'] = 0.1
# Efficiency reward - encourage purposeful movement
if not done and self.steps_since_food < 50: # Only apply when actively hunting
rewards['efficiency'] = self.reward_scales['efficiency'] * (1.0 - self.steps_since_food / 50)
# Calculate final reward - simple sum of components
reward = sum(rewards.values())
# Get observation
observation = self._get_normalized_observation()
# No more episode-based truncation
truncated = False
# Include reward components and exploration info in info dict for monitoring
info = {
"score": state["score"],
"reward_components": rewards,
"exploration_bonus": self.current_exploration_bonus,
"direction_changed": direction_changed,
"steps_in_same_direction": self.steps_in_same_direction,
"snake_length": snake_length,
"steps": self.steps # Include total steps for monitoring
}
return observation, reward, done, truncated, info
def render(self):
"""Render the environment."""
if self.window is None and self.render_mode == "human":
pygame.init()
self.window = pygame.display.set_mode((self.window_width, self.window_height))
pygame.display.set_caption("Snake Environment")
self.clock = pygame.time.Clock()
if self.window is not None:
self.window.fill((0, 0, 0))
# Draw game area
self.game_area.draw(self.window)
# Draw game session
self.session.render(self.window, self.game_area)
# Draw score
font = pygame.font.Font(None, 36)
score_text = f"Score: {self.session.score}"
score_surface = font.render(score_text, True, (0, 255, 0))
score_rect = score_surface.get_rect(midtop=(self.window_width // 2, 20))
self.window.blit(score_surface, score_rect)
pygame.display.flip()
self.clock.tick(self.metadata["render_fps"])
if self.render_mode == "rgb_array":
return np.transpose(
np.array(pygame.surfarray.pixels3d(self.window)),
axes=(1, 0, 2)
)
def close(self):
"""Close the environment."""
if self.window is not None:
pygame.quit()
self.window = None
def get_state(self) -> Dict:
"""Get the current game state."""
state = self.session.get_state()
state['steps_since_food'] = self.steps_since_food
# Calculate and include current time limit
snake_length = len(state["snake_body"])
current_time_limit = min(
self.max_time_limit,
self.base_time_limit + (snake_length - 1) * self.length_time_bonus
)
state['current_time_limit'] = current_time_limit
return state
def _get_adjacent_positions(self, position: Tuple[int, int]) -> List[Tuple[int, int]]:
"""
Get all adjacent positions to the given position.
Args:
position: Current position (x, y)
Returns:
List of adjacent positions [(x, y), ...]
Raises:
ValueError: If position is None or not a tuple of 2 integers
ValueError: If position coordinates are not integers
"""
if not position or not isinstance(position, tuple) or len(position) != 2:
raise ValueError("Position must be a tuple of 2 coordinates")
try:
x, y = int(position[0]), int(position[1])
except (TypeError, ValueError):
raise ValueError("Position coordinates must be integers")
# Get positions in all 4 directions (up, right, down, left)
adjacent = [
(x, y - 1), # up
(x + 1, y), # right
(x, y + 1), # down
(x - 1, y) # left
]
# Validate size attribute exists and is positive integer
if not hasattr(self, 'size') or not isinstance(self.size, int) or self.size <= 0:
raise ValueError("Environment size must be a positive integer")
# If wrap-around is enabled, adjust out-of-bounds positions
if hasattr(self, 'rules') and hasattr(self.rules, 'wrap_around') and self.rules.wrap_around:
adjacent = [
(pos[0] % self.size, pos[1] % self.size)
for pos in adjacent
]
return adjacent
# Otherwise, only return positions that are within bounds
return [
(pos[0], pos[1]) for pos in adjacent
if 0 <= pos[0] < self.size and 0 <= pos[1] < self.size
]
def _update_curriculum(self, score: int, done: bool) -> None:
"""Update curriculum stage based on agent's performance."""
if done:
if score >= self.stage_requirements[self.curriculum_stage]:
self.consecutive_successes += 1
else:
self.consecutive_successes = 0
# Check for stage advancement
if (self.curriculum_stage < 3 and
self.consecutive_successes >= self.success_threshold[self.curriculum_stage]):
self.curriculum_stage += 1
self.consecutive_successes = 0
# Increase episode length with curriculum stage
self.max_steps = self.base_max_steps * (1 + self.curriculum_stage * 0.5)
def _get_curriculum_rules(self) -> GameRules:
"""Get game rules appropriate for current curriculum stage."""
rules = GameRules()
# Adjust rules based on curriculum stage
if self.curriculum_stage == 0:
rules.wrap_around = True # Make it easier to avoid walls
rules.speed_increase = False
rules.initial_move_cooldown = 200 # Much slower movement
elif self.curriculum_stage == 1:
rules.wrap_around = True
rules.speed_increase = False # Still no speed increase
rules.initial_move_cooldown = 150
elif self.curriculum_stage == 2:
rules.wrap_around = True # Keep wrap-around until final stage
rules.speed_increase = True
rules.initial_move_cooldown = 120
else: # Stage 3
rules.wrap_around = False
rules.speed_increase = True
rules.initial_move_cooldown = 100
return rules
# Register the environment with Gymnasium
from gymnasium.envs.registration import register
register(
id='Snake-v0',
entry_point='src.ai.environment:SnakeEnv',
kwargs={
'size': 30,
'render_mode': None,
'difficulty': 'easy'
}
)
+93
View File
@@ -0,0 +1,93 @@
import gym
import numpy as np
import torch
import torch.nn as nn
from stable_baselines3.common.torch_layers import BaseFeaturesExtractor
class CustomRecurrentCNN(BaseFeaturesExtractor):
"""
A simple CNN with an LSTM layer added on top. This enables the network to keep an internal
memory of past observations so that it can adapt to changing rules/environment dynamics.
"""
def __init__(self, observation_space, features_dim=512):
# Initialize with a features_dim that matches the LSTM's output.
super(CustomRecurrentCNN, self).__init__(observation_space, features_dim)
# Calculate the expected CNN output size first
n_input_channels = observation_space.shape[0] # Number of input channels
self.cnn = nn.Sequential(
# Layer 1: (n_channels, 30, 30) -> (32, 15, 15)
nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=2, padding=1),
nn.ReLU(),
# Layer 2: (32, 15, 15) -> (64, 8, 8)
nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1),
nn.ReLU(),
# Layer 3: (64, 8, 8) -> (64, 4, 4)
nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1),
nn.ReLU(),
nn.Flatten()
)
# Calculate CNN output dimension using a dummy forward pass
with torch.no_grad():
dummy_input = torch.zeros(1, *observation_space.shape)
cnn_output = self.cnn(dummy_input)
self._cnn_output_dim = cnn_output.shape[1] # Should be 64 * 4 * 4 = 1024
# Add linear layer to reduce CNN output to desired feature dimension
self.fc = nn.Linear(self._cnn_output_dim, 256)
# LSTM layer
self.lstm = nn.LSTM(256, features_dim, batch_first=True)
def forward(self, observations):
# Pass through CNN: (batch, channels, height, width) -> (batch, cnn_features)
cnn_out = self.cnn(observations)
# Pass through linear layer: (batch, cnn_features) -> (batch, 256)
fc_out = self.fc(cnn_out)
# Add time dimension for LSTM: (batch, 256) -> (batch, 1, 256)
lstm_input = fc_out.unsqueeze(1)
# LSTM: (batch, 1, 256) -> (batch, 1, features_dim)
lstm_out, _ = self.lstm(lstm_input)
# Remove time dimension: (batch, 1, features_dim) -> (batch, features_dim)
return lstm_out.squeeze(1)
class CustomCNN(BaseFeaturesExtractor):
"""
Custom feature extractor for the snake environment.
Uses fully connected layers since our input is a 1D vector of normalized features.
"""
def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):
super().__init__(observation_space, features_dim)
self.input_dim = int(np.prod(observation_space.shape))
self.cnn = nn.Sequential(
nn.Linear(self.input_dim, 512),
nn.ReLU(),
nn.BatchNorm1d(512),
nn.Linear(512, 512),
nn.ReLU(),
nn.BatchNorm1d(512),
nn.Linear(512, features_dim),
nn.ReLU(),
)
def forward(self, observations: torch.Tensor) -> torch.Tensor:
# Handle both single and batch observations
if len(observations.shape) == 1:
observations = observations.unsqueeze(0)
# Ensure the input dimension matches what we expect
flat_obs = observations.reshape(observations.shape[0], self.input_dim)
return self.cnn(flat_obs)
+423
View File
@@ -0,0 +1,423 @@
"""
Training script for Snake AI using Stable Baselines3.
This script sets up and trains RL agents using the SnakeEnv environment.
It supports multiple difficulty levels and provides real-time visualization
of the training process.
"""
import os
import time
import numpy as np
import torch
from stable_baselines3 import PPO
from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv
from stable_baselines3.common.callbacks import BaseCallback
from stable_baselines3.common.utils import set_random_seed
from stable_baselines3.common.monitor import Monitor
import gymnasium as gym
import torch.nn as nn
import argparse
from queue import Queue
from threading import Thread
from typing import Dict, Any, Optional
import json
from queue import Empty
from src.ai.environment import SnakeEnv
from src.ai.visualize import run_dashboard
from src.ai.model_networks import CustomRecurrentCNN, CustomCNN
class VisualizationCallback(BaseCallback):
"""Callback for updating the training visualization."""
def __init__(self, viz_queue: Queue, total_timesteps: int, eval_freq: int = 1000, n_steps: int = 512, n_envs: int = 32, initial_timesteps: int = 0):
super().__init__()
self.viz_queue = viz_queue
self.total_timesteps = total_timesteps
self.eval_freq = eval_freq
self.n_steps = n_steps
self.n_envs = n_envs
self.initial_timesteps = initial_timesteps # Store initial timesteps
# Training metrics
self.episode_rewards = []
self.episode_lengths = []
# Evaluation metrics
self.eval_scores = []
self.high_score = 0
self.recent_high_score = 0
# Current episode tracking
self.current_reward = 0
self.current_length = 0
# Evaluation games (one with wrap, one without)
self.eval_envs = [
Monitor(SnakeEnv(render_mode="rgb_array")) for _ in range(2)
]
self.eval_envs[0].unwrapped.rules.wrap_around = False
self.eval_envs[1].unwrapped.rules.wrap_around = True
self.eval_obs = [None, None]
self.eval_done = [True, True]
self.eval_scores_current = [0, 0]
# Real-time demo environment
self.demo_env = Monitor(SnakeEnv(render_mode="rgb_array"))
self.demo_env.unwrapped.rules.wrap_around = True
self.demo_obs = None
self.demo_done = True
self.last_demo_time = 0
self.demo_speed = 50
def _on_step(self) -> bool:
"""Update visualization on each step."""
try:
current_time = time.time() * 1000 # Convert to milliseconds
# Update episode tracking
self.current_length += 1
reward = self.locals.get("rewards", [0])[0]
self.current_reward += reward
# Get action probabilities and value estimate for demo game
if self.demo_obs is not None:
# Ensure proper tensor shape (batch_size, obs_dim)
demo_obs_tensor = torch.as_tensor(self.demo_obs).float()
if len(demo_obs_tensor.shape) == 1:
demo_obs_tensor = demo_obs_tensor.unsqueeze(0)
# Move tensor to same device as model's policy
device = self.model.policy.device
demo_obs_tensor = demo_obs_tensor.to(device)
with torch.no_grad():
# Get action distribution and value estimate
dist = self.model.policy.get_distribution(demo_obs_tensor)
action_probs = dist.distribution.probs
value_estimate = self.model.policy.predict_values(demo_obs_tensor)
# Move results back to CPU and get first item (since we added batch dimension)
action_probs = action_probs[0].cpu().numpy()
value_estimate = value_estimate[0].cpu().numpy()
else:
action_probs = np.zeros(4)
value_estimate = np.array([0.0])
# Check if episode ended
dones = self.locals.get("dones", [False])
if any(dones):
self.episode_rewards.append(self.current_reward)
self.episode_lengths.append(self.current_length)
self.current_reward = 0
self.current_length = 0
# Run evaluation games (keep these deterministic as they're meant to show best performance)
for i, (env, obs, done) in enumerate(zip(self.eval_envs, self.eval_obs, self.eval_done)):
if done:
if obs is not None: # If this wasn't the first reset
self.eval_scores.append(self.eval_scores_current[i])
# Update high scores based on evaluation performance
self.high_score = max(self.high_score, self.eval_scores_current[i])
self.recent_high_score = max(self.recent_high_score, self.eval_scores_current[i])
self.eval_obs[i] = env.reset()[0]
self.eval_done[i] = False
self.eval_scores_current[i] = 0
eval_action, _ = self.model.predict(self.eval_obs[i], deterministic=True)
eval_action = eval_action.item() if hasattr(eval_action, 'item') else eval_action
self.eval_obs[i], _, done, truncated, info = env.step(eval_action)
self.eval_done[i] = done or truncated
# Update current evaluation score
self.eval_scores_current[i] = info.get("score", 0)
# Run real-time demo game
if current_time - self.last_demo_time >= self.demo_speed:
if self.demo_done:
self.demo_obs = self.demo_env.reset()[0]
self.demo_done = False
demo_action, _ = self.model.predict(self.demo_obs, deterministic=True)
demo_action = demo_action.item() if hasattr(demo_action, 'item') else demo_action
self.demo_obs, _, done, truncated, info = self.demo_env.step(demo_action)
self.demo_done = done or truncated
self.last_demo_time = current_time
# Get current states from actual training environments
# Sample one no-wrap and one wrap environment from the training environments
no_wrap_idx = 0 # First half are no-wrap
wrap_idx = self.n_envs // 2 # Second half are wrap
training_state_1 = self.training_env.get_attr("session")[no_wrap_idx].get_state()
training_state_1["wrap_around"] = False
training_state_2 = self.training_env.get_attr("session")[wrap_idx].get_state()
training_state_2["wrap_around"] = True
eval_state_1 = self.eval_envs[0].unwrapped.session.get_state()
eval_state_1["wrap_around"] = self.eval_envs[0].unwrapped.rules.wrap_around
eval_state_2 = self.eval_envs[1].unwrapped.session.get_state()
eval_state_2["wrap_around"] = self.eval_envs[1].unwrapped.rules.wrap_around
demo_state = self.demo_env.unwrapped.session.get_state()
demo_state["wrap_around"] = self.demo_env.unwrapped.rules.wrap_around
# Update viz_data
viz_data = {
'training_state': [training_state_1, training_state_2],
'eval_states': [eval_state_1, eval_state_2],
'demo_state': demo_state,
'weights_info': {
'action_probs': action_probs.tolist() if 'action_probs' in locals() else [0.25] * 4,
'value_estimate': float(value_estimate[0]) if 'value_estimate' in locals() else 0.0,
'action_labels': ['Up', 'Right', 'Down', 'Left']
},
'training_info': {
'total_timesteps': self.num_timesteps,
'initial_timesteps': self.initial_timesteps, # Add initial timesteps
'target_timesteps': self.total_timesteps,
'episode_reward': float(self.current_reward),
'mean_reward': float(np.mean(self.episode_rewards[-100:])) if self.episode_rewards else 0.0,
'episode_length': int(self.current_length),
'mean_length': float(np.mean(self.episode_lengths[-100:])) if self.episode_lengths else 0.0,
'mean_eval_score': float(np.mean(self.eval_scores[-100:])) if self.eval_scores else 0.0,
'rewards_history': self.episode_rewards[-1000:],
'lengths_history': self.episode_lengths[-1000:],
'eval_scores_history': self.eval_scores[-1000:],
'high_score': self.high_score,
'recent_high_score': self.recent_high_score
}
}
# Send update to visualization
self.viz_queue.put(('update', viz_data))
# Reset recent high score periodically
if self.num_timesteps % (self.n_steps * self.n_envs) == 0:
print(f"\nEvaluation High Score: {self.high_score}")
print(f"Recent Evaluation High Score: {self.recent_high_score}")
self.recent_high_score = 0
return True
except Exception as e:
print(f"Error in visualization callback: {e}")
import traceback
traceback.print_exc()
return True
def make_env(seed: int = 0, wrap_walls: bool = False) -> gym.Env:
"""Create a training environment.
Args:
seed: Random seed
wrap_walls: Whether snake can wrap around walls
"""
def _init() -> gym.Env:
env = SnakeEnv(render_mode=None)
env = Monitor(env)
# Set wrap-around mode
env.unwrapped.rules.wrap_around = wrap_walls
env.reset(seed=seed)
return env
set_random_seed(seed)
return _init
def train_model(
total_timesteps: int,
viz_queue: Queue,
model_path: Optional[str] = None,
cuda: bool = True,
learning_rate: float = 3e-4, # Standard PPO learning rate
batch_size: int = 256, # More reasonable batch size
n_envs: int = 16, # Balanced number of environments
n_steps: int = 2048 # Standard PPO steps
) -> None:
"""
Train the snake AI model.
Args:
total_timesteps: Number of timesteps to train for
viz_queue: Queue for sending visualization updates
model_path: Path to save/load the model
cuda: Whether to use CUDA for training
learning_rate: Learning rate for the optimizer
batch_size: Batch size for training
n_envs: Number of parallel environments
n_steps: Number of steps per environment
"""
try:
# Set up environments - half with wrap, half without
envs = []
for i in range(n_envs):
wrap_walls = i >= n_envs // 2 # Half the envs with wrap-around
envs.append(make_env(i, wrap_walls))
env = SubprocVecEnv(envs)
# PPO hyperparameters - adjusted for more stable learning
ppo_params = dict(
learning_rate=1e-4, # Slightly higher learning rate
n_steps=1024, # Shorter horizon for faster updates
batch_size=128, # Smaller batch size for more frequent updates
n_epochs=4, # Balanced number of epochs
gamma=0.98, # Slightly lower discount for more immediate rewards
gae_lambda=0.9, # Lower GAE lambda for more immediate advantage estimates
clip_range=0.1, # Smaller clip range for more conservative updates
clip_range_vf=0.1, # Also clip value function
normalize_advantage=True,
ent_coef=0.2, # Moderate entropy for exploration
vf_coef=0.8, # Balanced value function coefficient
max_grad_norm=0.3, # More conservative gradient clipping
target_kl=None,
verbose=1,
device="cuda" if cuda and torch.cuda.is_available() else "cpu"
)
# Create or load model with updated architecture
policy_kwargs = dict(
features_extractor_class=CustomRecurrentCNN, # Use the new recurrent extractor
features_extractor_kwargs=dict(features_dim=512), # You can adjust features_dim as needed
net_arch=dict(
pi=[512, 256, 128], # Policy network layers
vf=[512, 256, 128] # Value network layers
),
activation_fn=nn.ReLU,
normalize_images=False
)
# Track initial timesteps for progress tracking
initial_timesteps = 0
if model_path and os.path.exists(model_path):
print(f"Loading model from {model_path}")
try:
model = PPO.load(
model_path,
env=env,
custom_objects={"policy_kwargs": policy_kwargs},
**ppo_params
)
initial_timesteps = model.num_timesteps
print(f"Loaded model with {initial_timesteps:,} timesteps of training")
except Exception as e:
print(f"Failed to load model with error: {e}")
print("Creating new model instead")
model = PPO("CnnPolicy", env, policy_kwargs=policy_kwargs, **ppo_params)
else:
print("Creating new model")
model = PPO("CnnPolicy", env, policy_kwargs=policy_kwargs, **ppo_params)
# Set up visualization callback with initial timesteps
viz_callback = VisualizationCallback(
viz_queue=viz_queue,
total_timesteps=total_timesteps,
eval_freq=1000,
n_steps=n_steps,
n_envs=n_envs,
initial_timesteps=initial_timesteps # Pass initial timesteps to callback
)
# Training loop with stop check
should_stop = False
timesteps_per_batch = n_steps * n_envs
num_batches = total_timesteps // timesteps_per_batch
for i in range(num_batches):
if should_stop:
break
# Train for one batch
model.learn(
total_timesteps=timesteps_per_batch,
callback=viz_callback,
progress_bar=True,
reset_num_timesteps=False
)
# Check for stop signal from visualization
try:
while True: # Process all pending messages
msg_type, _ = viz_queue.get_nowait()
if msg_type == 'stop_training':
should_stop = True
break
except Empty:
pass
# Save model if training completed or interrupted
if model_path:
print(f"Saving model to {model_path}")
model.save(model_path)
except Exception as e:
print(f"Error in training process: {e}")
import traceback
traceback.print_exc()
finally:
# Clean up
env.close()
# Signal visualization to stop if we haven't already
viz_queue.put(('stop', None))
def main():
"""Main training script."""
parser = argparse.ArgumentParser(description="Train Snake AI")
parser.add_argument("--config", action="store_true",
help="Show configuration UI before training")
parser.add_argument("--model", type=str, default="models/snake_ai.zip",
help="Path to save/load model")
parser.add_argument("--no-cuda", action="store_true",
help="Disable CUDA training")
args = parser.parse_args()
# Get training configuration
if args.config:
from src.ai.config_ui import ConfigUI
config_ui = ConfigUI()
config = config_ui.run()
if config is None: # User cancelled
return
else:
# Load saved config or use defaults
config_file = "training_config.json"
if os.path.exists(config_file):
with open(config_file, 'r') as f:
config = json.load(f)
else:
config = {
'timesteps': 2000000,
'learning_rate': 0.0001,
'batch_size': 512,
'n_envs': 32,
'n_steps': 512
}
# Create visualization queue
viz_queue = Queue()
# Start visualization in separate thread
viz_thread = Thread(target=run_dashboard, args=(viz_queue,))
viz_thread.start()
# Start training with configuration
train_model(
total_timesteps=config['timesteps'],
viz_queue=viz_queue,
model_path=args.model,
cuda=not args.no_cuda,
learning_rate=config['learning_rate'],
batch_size=config['batch_size'],
n_envs=config['n_envs'],
n_steps=config['n_steps']
)
# Wait for visualization to finish
viz_thread.join()
if __name__ == "__main__":
main()
+540
View File
@@ -0,0 +1,540 @@
"""
Training Visualization Dashboard
This module provides a real-time visualization of the training process,
showing both the training metrics and evaluation games.
"""
import pygame
import numpy as np
from typing import Dict, Any, Optional, List, Tuple
import threading
from queue import Queue, Empty
import time
class TrainingDashboard:
def __init__(self, width: int = 1920, height: int = 1080):
"""Initialize the training dashboard."""
self.width = width
self.height = height
# Fonts
self.title_font = None
self.header_font = None
self.text_font = None
# Colors
self.colors = {
'background': (0, 0, 0),
'text': (200, 200, 200),
'highlight': (0, 255, 0),
'grid': (40, 40, 40),
'border': (0, 255, 0),
'snake': (0, 255, 0),
'food': (255, 0, 0),
'graph_bg': (20, 20, 20),
'graph_line': (0, 255, 0),
'graph_grid': (40, 40, 40)
}
# Layout
self.layout = {
'margin': 20,
'game_size': 400,
'graph_height': 200,
'metrics_width': 300
}
# Training metrics history
self.metrics_history = {
'rewards': [],
'scores': [],
'lengths': []
}
self.max_history = 1000
# Current state
self.current_state = None
self.eval_state = None
self.demo_state = None
self.training_info = None
self.model_info = None
# Performance tracking
self.frame_times = []
self.max_frame_times = 60
# Initialize the clock
self.clock = None
self.target_fps = 60
def run(self, update_queue: Queue) -> None:
"""Run the dashboard, processing updates from the queue."""
pygame.init()
try:
self.screen = pygame.display.set_mode((self.width, self.height))
pygame.display.set_caption("Snake AI Training Dashboard")
# Initialize fonts after pygame is initialized
self.title_font = pygame.font.Font(None, 48)
self.header_font = pygame.font.Font(None, 36)
self.text_font = pygame.font.Font(None, 24)
# Initialize clock
self.clock = pygame.time.Clock()
running = True
while running:
# Handle events
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
elif event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
running = False
# Process updates from queue
try:
while True: # Process all available updates
update_type, data = update_queue.get_nowait()
if update_type == 'update':
self.update(data)
elif update_type == 'stop':
running = False
break
except Empty:
pass
# Render dashboard
self.render()
self.clock.tick(self.target_fps)
finally:
pygame.quit()
def update(self, data: Dict[str, Any]) -> None:
"""Update the dashboard with new data."""
self.current_state = data # Store entire data dict
if 'training_info' in data:
self.training_info = data['training_info']
# Update metrics history
for key in ['rewards', 'scores', 'lengths']:
if key in self.training_info:
self.metrics_history[key].append(self.training_info[key])
if len(self.metrics_history[key]) > self.max_history:
self.metrics_history[key].pop(0)
def draw_game_view(self, state: Dict[str, Any], position: Tuple[int, int], size: int, title: str) -> None:
"""Draw a game view at the specified position."""
if not state:
return
x, y = position
# Draw game area background
game_rect = pygame.Rect(x, y, size, size)
pygame.draw.rect(self.screen, self.colors['grid'], game_rect)
pygame.draw.rect(self.screen, self.colors['border'], game_rect, 2)
# Get time limit info directly from state
steps_since_food = state.get('steps_since_food', 0)
current_time_limit = state.get('current_time_limit', 100) # Default to base limit if not provided
time_left = max(0, current_time_limit - steps_since_food)
# Calculate cell size based on grid dimensions
cell_size = size // 30 # Assuming 30x30 grid
# Draw snake
if "snake_body" in state:
for segment in state["snake_body"]:
segment_rect = pygame.Rect(
x + segment[0] * cell_size,
y + segment[1] * cell_size,
cell_size, cell_size
)
pygame.draw.rect(self.screen, self.colors['snake'], segment_rect)
# Draw food
if "food_position" in state:
food_rect = pygame.Rect(
x + state["food_position"][0] * cell_size,
y + state["food_position"][1] * cell_size,
cell_size, cell_size
)
pygame.draw.rect(self.screen, self.colors['food'], food_rect)
# Draw score and timer above game area
score = state.get('score', 0)
score_text = f"Score: {score}"
timer_text = f"Time: {time_left}/{current_time_limit}" # Show both current and max time
combined_text = f"{score_text} | {timer_text}"
score_surface = self.text_font.render(combined_text, True, self.colors['text'])
score_rect = score_surface.get_rect(midtop=(x + size//2, y - 25))
self.screen.blit(score_surface, score_rect)
# Draw title below game area, including wrap mode for training view
if title == "Training Episode" and 'wrap_around' in state:
wrap_mode = "(Wrap)" if state['wrap_around'] else "(No Wrap)"
title = f"{title} {wrap_mode}"
title_text = title
title_surface = self.header_font.render(title_text, True, self.colors['text'])
title_rect = title_surface.get_rect(midtop=(x + size//2, y + size + 10))
self.screen.blit(title_surface, title_rect)
def draw_metrics(self, x: int, y: int, width: int) -> None:
"""Draw training metrics."""
if not self.training_info:
return
font = pygame.font.Font(None, 24)
y_offset = 0
# Draw high scores
high_score = self.training_info.get("high_score", 0)
recent_high = self.training_info.get("recent_high_score", 0)
metrics = [
("All-time High Score", high_score),
("Recent High Score", recent_high),
("", ""), # Empty line for spacing
("Training Metrics", ""),
("Total Steps", self.training_info.get('total_timesteps', 0)),
("Episode Reward", f"{self.training_info.get('episode_reward', 0):.2f}"),
("Mean Reward", f"{self.training_info.get('mean_reward', 0):.2f}"),
("Episode Length", self.training_info.get('episode_length', 0)),
("Mean Length", f"{self.training_info.get('mean_length', 0):.2f}"),
("FPS", f"{len(self.frame_times) / sum(self.frame_times):.1f}" if self.frame_times else "0")
]
for i, (label, value) in enumerate(metrics):
text = f"{label}: {value}" if label else ""
surface = self.text_font.render(text, True, self.colors['text'])
self.screen.blit(surface, (x, y + i * 25))
def draw_graph(self, data: List[float], rect: pygame.Rect, title: str,
color: Tuple[int, int, int], min_val: float, max_val: float,
smoothing_window: int = 10) -> None:
"""Draw a line graph of the given data.
Args:
data: List of data points.
rect: Rectangle area for drawing the graph.
title: Title for the graph.
color: Color for the graph line.
min_val: Minimum value for scaling the y-axis.
max_val: Maximum value for scaling the y-axis.
smoothing_window: Optional smoothing window size for drawing
long term trends. Increase this value for a smoother, longer-term trend.
Default is 10.
"""
if not data:
return
# Create a separate surface for the graph
graph_surface = pygame.Surface((rect.width, rect.height))
graph_surface.fill(self.colors['graph_bg'])
# Draw grid lines on the graph surface
num_lines = 5
for i in range(num_lines):
y_line = i * rect.height / (num_lines - 1)
pygame.draw.line(graph_surface, self.colors['graph_grid'],
(0, y_line), (rect.width, y_line))
# Add padding to the min/max values for better visualization
value_range = max_val - min_val
if value_range == 0:
value_range = 1.0
padding = value_range * 0.1
min_val_adjusted = min_val - padding
max_val_adjusted = max_val + padding
# Smooth the data using the provided smoothing_window parameter
window_size = smoothing_window if len(data) >= smoothing_window else len(data)
if window_size > 1:
smoothed_data = np.convolve(data, np.ones(window_size) / window_size, mode='valid')
else:
smoothed_data = data
points = []
for i, value in enumerate(smoothed_data):
x_point = i * rect.width / len(smoothed_data)
y_point = rect.height - ((value - min_val_adjusted) * rect.height /
(max_val_adjusted - min_val_adjusted))
points.append((x_point, y_point))
if len(points) > 1:
pygame.draw.lines(graph_surface, color, False, points, 2)
# Draw border on the graph surface
pygame.draw.rect(graph_surface, self.colors['border'],
pygame.Rect(0, 0, rect.width, rect.height), 1)
# Blit the graph surface onto the main screen
self.screen.blit(graph_surface, rect)
# Draw title and current value
current_value = data[-1] if data else 0
title_text = f"{title} (Current: {current_value:.2f})"
title_surface = self.text_font.render(title_text, True, self.colors['text'])
title_rect = title_surface.get_rect(midtop=(rect.centerx, rect.top - 20))
self.screen.blit(title_surface, title_rect)
# Draw value labels to the right of the graph
label_spacing = rect.height / (num_lines - 1)
label_x = rect.right + 10
for i in range(num_lines):
value = max_val_adjusted - (i * (max_val_adjusted - min_val_adjusted) / (num_lines - 1))
label_text = f"{value:.1f}"
label_surface = self.text_font.render(label_text, True, self.colors['text'])
label_rect = label_surface.get_rect(
left=label_x,
centery=rect.top + i * label_spacing
)
self.screen.blit(label_surface, label_rect)
def draw_weights_visualization(self, x: int, y: int, width: int, height: int, weights_info: Dict) -> None:
"""Draw the neural network weights visualization."""
if not weights_info:
return
# Get data
action_probs = weights_info.get('action_probs', [0, 0, 0, 0])
value_estimate = weights_info.get('value_estimate', 0.0)
action_labels = weights_info.get('action_labels', ['Up', 'Right', 'Down', 'Left'])
# Draw background
rect = pygame.Rect(x, y, width, height)
pygame.draw.rect(self.screen, self.colors['graph_bg'], rect)
pygame.draw.rect(self.screen, self.colors['border'], rect, 1)
# Draw title
title = "Action Probabilities"
title_surface = self.header_font.render(title, True, self.colors['text'])
title_rect = title_surface.get_rect(midtop=(x + width//2, y + 5))
self.screen.blit(title_surface, title_rect)
# Draw action probability bars
bar_height = 20
bar_spacing = 30
bar_start_y = y + 50
max_bar_width = width - 120 # Leave space for labels
for i, (prob, label) in enumerate(zip(action_probs, action_labels)):
# Draw label
label_surface = self.text_font.render(label, True, self.colors['text'])
self.screen.blit(label_surface, (x + 10, bar_start_y + i * bar_spacing))
# Draw bar background
bar_bg_rect = pygame.Rect(x + 70, bar_start_y + i * bar_spacing, max_bar_width, bar_height)
pygame.draw.rect(self.screen, self.colors['grid'], bar_bg_rect)
# Draw probability bar
bar_width = int(prob * max_bar_width)
bar_rect = pygame.Rect(x + 70, bar_start_y + i * bar_spacing, bar_width, bar_height)
pygame.draw.rect(self.screen, self.colors['highlight'], bar_rect)
# Draw probability value
prob_text = f"{prob:.2f}"
prob_surface = self.text_font.render(prob_text, True, self.colors['text'])
self.screen.blit(prob_surface, (x + 80 + max_bar_width, bar_start_y + i * bar_spacing))
# Draw value estimate
value_text = f"Value Estimate: {value_estimate:.2f}"
value_surface = self.text_font.render(value_text, True, self.colors['text'])
value_rect = value_surface.get_rect(midbottom=(x + width//2, y + height - 10))
self.screen.blit(value_surface, value_rect)
def draw_progress_bar(self, x: int, y: int, width: int, height: int, progress: float, text: str) -> None:
"""Draw a progress bar with text.
Args:
x, y: Position of the progress bar
width, height: Dimensions of the progress bar
progress: Progress value between 0 and 1
text: Text to display above the progress bar
"""
# Draw background
bg_rect = pygame.Rect(x, y, width, height)
pygame.draw.rect(self.screen, self.colors['graph_bg'], bg_rect)
pygame.draw.rect(self.screen, self.colors['border'], bg_rect, 1)
# Draw progress
if progress > 0:
progress_width = int(width * progress)
progress_rect = pygame.Rect(x, y, progress_width, height)
pygame.draw.rect(self.screen, self.colors['highlight'], progress_rect)
# Draw text
text_surface = self.text_font.render(text, True, self.colors['text'])
text_rect = text_surface.get_rect(bottomleft=(x, y - 5))
self.screen.blit(text_surface, text_rect)
def render(self) -> None:
"""Render the dashboard."""
if not self.current_state:
return
self.screen.fill(self.colors['background'])
# Draw game views
if 'training_state' in self.current_state:
# Draw both training games
self.draw_game_view(
self.current_state['training_state'][0], # No wrap training
(50, 50),
250,
"Training (No Wrap)"
)
self.draw_game_view(
self.current_state['training_state'][1], # Wrap training
(350, 50),
250,
"Training (Wrap)"
)
if 'eval_states' in self.current_state:
self.draw_game_view(
self.current_state['eval_states'][0],
(650, 50),
250,
"Evaluation (No Wrap)"
)
self.draw_game_view(
self.current_state['eval_states'][1],
(950, 50),
250,
"Evaluation (Wrap)"
)
if 'demo_state' in self.current_state:
self.draw_game_view(
self.current_state['demo_state'],
(1250, 50),
250,
"Demo Game"
)
# Draw metrics
if self.training_info:
metrics_data = [
(self.training_info['rewards_history'], "Episode Rewards"),
(self.training_info['lengths_history'], "Episode Lengths"),
(self.training_info['eval_scores_history'], "Evaluation Scores")
]
for i, (data, label) in enumerate(metrics_data):
if data: # Only draw if we have data
rect = pygame.Rect(50, 400 + i * 200, 500, 150)
self.draw_graph(
data,
rect,
label,
self.colors['graph_line'],
min(data) if data else 0,
max(data) if data else 1
)
# Draw current metrics text
metrics_text = [
f"Total Steps: {self.training_info.get('total_timesteps', 0):,}",
f"Mean Reward: {self.training_info.get('mean_reward', 0):.2f}",
f"Mean Length: {self.training_info.get('mean_length', 0):.1f}",
f"Mean Eval Score: {self.training_info.get('mean_eval_score', 0):.2f}",
f"High Score: {self.training_info.get('high_score', 0)}",
f"Recent High Score: {self.training_info.get('recent_high_score', 0)}"
]
# Add vertical spacer between metrics sections
metrics_text.insert(3, "") # Insert empty string as spacer after first 3 metrics
font = pygame.font.Font(None, 36)
for i, text in enumerate(metrics_text):
surface = font.render(text, True, self.colors['text'])
self.screen.blit(surface, (600, 400 + i * 30))
# Draw action probabilities if available
if 'weights_info' in self.current_state:
self.draw_weights_visualization(
1100, 400,
300, 300,
self.current_state['weights_info']
)
# Draw trend indicator graph
if self.training_info:
# Calculate trend indicators
window = 100 # Use last 100 episodes for trends
rewards = self.training_info['rewards_history'][-window:]
lengths = self.training_info['lengths_history'][-window:]
eval_scores = self.training_info['eval_scores_history'][-window:]
if rewards and lengths and eval_scores:
# Normalize each metric to 0-1 range for fair combination
norm_rewards = [(r - min(rewards)) / (max(rewards) - min(rewards) + 1e-8) for r in rewards]
norm_lengths = [(l - min(lengths)) / (max(lengths) - min(lengths) + 1e-8) for l in lengths]
norm_scores = [(s - min(eval_scores)) / (max(eval_scores) - min(eval_scores) + 1e-8) for s in eval_scores]
# Combine metrics with weights (rewards: 0.4, lengths: 0.3, eval_scores: 0.3)
trend_data = [0.4 * r + 0.3 * l + 0.3 * s
for r, l, s in zip(norm_rewards, norm_lengths, norm_scores)]
# Add trend direction indicator
trending = 0 # -1, 0, 1
if len(trend_data) >= 2:
recent_trend = trend_data[-1] - trend_data[0]
trending = 1 if recent_trend > 0.1 else -1 if recent_trend < -0.1 else 0
trend_text = "↑ Improving" if trending == 1 else "↓ Declining" if trending == -1 else "→ Stable"
trend_color = (0, 255, 0) if trending == 1 else (255, 0, 0) if trending == -1 else (200, 200, 0)
trend_surface = self.text_font.render(trend_text, True, trend_color)
trend_rect = trend_surface.get_rect(midtop=(1250, 860))
self.screen.blit(trend_surface, trend_rect)
graph_color = (0, 255, 0) if trending == 1 else (255, 0, 0) if trending == -1 else (200, 200, 0)
# Draw trend graph 50 pixels below the action probabilities graph
trend_rect = pygame.Rect(1100, 750, 300, 150)
self.draw_graph(
trend_data,
trend_rect,
"Learning Trend",
graph_color, # color based on trend
0, # Min value (normalized)
1, # Max value (normalized)
smoothing_window=10 # Use a larger smoothing window for longer-term trends
)
# Draw training progress bar at the bottom of the screen
if self.training_info:
# Get current session steps (from start of this training run)
current_session_steps = self.training_info.get('total_timesteps', 0) - self.training_info.get('initial_timesteps', 0)
target_timesteps = self.training_info.get('target_timesteps', 1000000)
progress = min(1.0, current_session_steps / target_timesteps)
# Show both current session progress and total model training
total_model_steps = self.training_info.get('total_timesteps', 0)
progress_text = (
f"Current Session: {current_session_steps:,}/{target_timesteps:,} steps ({progress*100:.1f}%) | "
f"Total Model Training: {total_model_steps:,} steps"
)
self.draw_progress_bar(50, self.height - 40, self.width - 100, 20, progress, progress_text)
pygame.display.flip()
def handle_events(self):
for event in pygame.event.get():
if event.type == pygame.QUIT:
self.running = False
elif event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
self.running = False
def run_dashboard(update_queue: Queue) -> None:
"""Run the training dashboard in a separate thread."""
dashboard = TrainingDashboard()
try:
dashboard.run(update_queue)
finally:
# Send stop signal back to training process
update_queue.put(('stop_training', None))
pygame.quit()