diff --git a/src/compute/src/agents.rs b/src/compute/src/agents.rs index 77d77c7..581e4cf 100644 --- a/src/compute/src/agents.rs +++ b/src/compute/src/agents.rs @@ -1,7 +1,9 @@ +mod mcts; mod metrics; mod minimax; mod stats; +pub use mcts::{MctsAgent, MctsStats, MctsStatsAccumulator}; pub use metrics::*; pub use minimax::{HeuristicEvaluator, MinimaxAgent, MinimaxStats, MinimaxStatsAccumulator}; pub use stats::{AgentRuntimeStats, AgentStatsAccumulator}; @@ -27,6 +29,16 @@ fn default_edge_penalty() -> i32 { -2 } +fn default_max_iterations() -> u32 { + 75000 +} +fn default_max_time_ms() -> Option { + None +} +fn default_exploration_constant() -> f64 { + 1.41 +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(tag = "type")] pub enum AgentConfig { @@ -46,6 +58,14 @@ pub enum AgentConfig { #[serde(default = "default_edge_penalty")] edge_penalty_weight: i32, }, + Mcts { + #[serde(default = "default_max_iterations")] + max_iterations: u32, + #[serde(default = "default_max_time_ms")] + max_time_ms: Option, + #[serde(default = "default_exploration_constant")] + exploration_constant: f64, + }, Human, } @@ -53,6 +73,7 @@ impl From<&AgentConfig> for AgentType { fn from(config: &AgentConfig) -> Self { match config { AgentConfig::Minimax { .. } => Self::Minimax, + AgentConfig::Mcts { .. } => Self::Mcts, AgentConfig::Human => Self::Human, } } @@ -76,6 +97,16 @@ impl AgentConfig { ); BreakthroughAgent::Minimax(MinimaxAgent::new(*max_depth, seed, Box::new(heuristic))) } + Self::Mcts { + max_iterations, + max_time_ms, + exploration_constant, + } => BreakthroughAgent::Mcts(MctsAgent::new( + *max_iterations, + *max_time_ms, + *exploration_constant, + seed, + )), Self::Human => BreakthroughAgent::Human, } } @@ -90,6 +121,7 @@ pub enum AgentStats { #[default] None, Minimax(MinimaxStats), + Mcts(MctsStats), } pub trait Agent: Send { @@ -101,6 +133,7 @@ pub trait Agent: Send { #[derive(Debug)] pub enum BreakthroughAgent { Minimax(MinimaxAgent), + Mcts(MctsAgent), Human, } @@ -108,6 +141,7 @@ impl BreakthroughAgent { pub fn select_ply(&mut self, board: &Board, config: &BoardConfig) -> Option { match self { Self::Minimax(agent) => agent.select_ply(board, config), + Self::Mcts(agent) => agent.select_ply(board, config), Self::Human => None, } } @@ -115,6 +149,7 @@ impl BreakthroughAgent { pub fn take_stats(&mut self) -> AgentStats { match self { Self::Minimax(agent) => agent.take_stats(), + Self::Mcts(agent) => agent.take_stats(), Self::Human => AgentStats::None, } } diff --git a/src/compute/src/agents/mcts.rs b/src/compute/src/agents/mcts.rs new file mode 100644 index 0000000..f497523 --- /dev/null +++ b/src/compute/src/agents/mcts.rs @@ -0,0 +1,210 @@ +#![allow(dead_code)] + +mod metrics; +pub mod node; +mod stats; + +use node::MctsNode; +pub use stats::{MctsStats, MctsStatsAccumulator}; + +use rand::rngs::SmallRng; +use rand::{RngExt, SeedableRng, seq::SliceRandom}; +use std::time::Instant; + +use super::{Agent, AgentStats}; +use crate::core::{Board, BoardConfig, Ply, Status}; + +#[derive(Debug)] +pub struct MctsAgent { + pub max_iterations: u32, + pub max_time_ms: Option, + pub exploration_constant: f64, + pub rng: SmallRng, + pub current_stats: MctsStats, +} + +impl MctsAgent { + pub fn new( + max_iterations: u32, + max_time_ms: Option, + exploration_constant: f64, + seed: u64, + ) -> Self { + Self { + max_iterations, + max_time_ms, + exploration_constant, + rng: SmallRng::seed_from_u64(seed), + current_stats: MctsStats::default(), + } + } + + fn check_limits(&self, iterations: u32, start_time: Instant) -> bool { + match self.max_time_ms { + Some(max_time) => { + if (iterations & 511) == 0 { + (start_time.elapsed().as_millis() as u64) < max_time + } else { + true + } + } + None => iterations < self.max_iterations, + } + } +} + +impl Agent for MctsAgent { + fn select_ply(&mut self, board: &Board, config: &BoardConfig) -> Option { + self.current_stats = MctsStats::default(); + let start_time = Instant::now(); + + let use_time = self.max_time_ms.is_some(); + let capacity = if use_time { + 100_000 + } else { + self.max_iterations as usize + 1 + }; + + let mut tree: Vec = Vec::with_capacity(capacity); + + let mut root_plies = board.get_legal_plies(config); + if root_plies.is_empty() { + return None; + } + + root_plies.shuffle(&mut self.rng); + + tree.push(MctsNode::new(None, None, root_plies, board.turn)); + self.current_stats.nodes_created += 1; + + while self.check_limits(self.current_stats.iterations, start_time) { + let mut current_idx = 0; + let mut sim_board = *board; + + // Phase 1: Selection + while tree[current_idx].unexpanded_plies.is_empty() + && !tree[current_idx].children.is_empty() + { + let parent_visits = tree[current_idx].visits; + let c = self.exploration_constant; + let mut best_child_idx = 0; + let mut best_score = f64::NEG_INFINITY; + + for &child_idx in &tree[current_idx].children { + let child = &tree[child_idx]; + let score = if child.visits == 0 { + f64::INFINITY + } else { + let exploitation = child.wins / (child.visits as f64); + let exploration = + c * ((parent_visits as f64).ln() / (child.visits as f64)).sqrt(); + exploitation + exploration + }; + + if score == f64::INFINITY { + best_child_idx = child_idx; + break; + } + + if score > best_score { + best_score = score; + best_child_idx = child_idx; + } + } + + current_idx = best_child_idx; + if let Some(ply) = tree[current_idx].ply_to_reach { + sim_board.apply_ply(ply); + } + } + + // Phase 2: Expansion + let mut status = sim_board.get_status(config); + + if status == Status::Ongoing && !tree[current_idx].unexpanded_plies.is_empty() { + let ply_to_expand = tree[current_idx].unexpanded_plies.pop().unwrap(); + sim_board.apply_ply(ply_to_expand); + + let mut next_plies = sim_board.get_legal_plies(config); + next_plies.shuffle(&mut self.rng); + let new_node = MctsNode::new( + Some(current_idx), + Some(ply_to_expand), + next_plies, + sim_board.turn, + ); + + let new_child_idx = tree.len(); + tree.push(new_node); + tree[current_idx].children.push(new_child_idx); + + current_idx = new_child_idx; + self.current_stats.nodes_created += 1; + status = sim_board.get_status(config); + } + + // Phase 3: Playout + let mut playout_steps = 0; + while status == Status::Ongoing { + let plies = sim_board.get_legal_plies(config); + if plies.is_empty() { + // Should not happen, but just in case + status = if sim_board.turn.is_white() { + Status::BlackWon + } else { + Status::WhiteWon + }; + break; + } + let random_idx = self.rng.random_range(0..plies.len()); + sim_board.apply_ply(plies[random_idx]); + status = sim_board.get_status(config); + playout_steps += 1; + } + self.current_stats.playout_steps += playout_steps; + + // Phase 4: Backpropagation + let mut node_ptr = Some(current_idx); + + while let Some(idx) = node_ptr { + let node = &mut tree[idx]; + node.visits += 1; + + let player_who_just_moved = node.turn.opponent(); + + let is_win = match status { + Status::WhiteWon => player_who_just_moved.is_white(), + Status::BlackWon => !player_who_just_moved.is_white(), + Status::Ongoing => false, + }; + + if is_win { + node.wins += 1.0; + } + + node_ptr = node.parent; + } + + self.current_stats.iterations += 1; + } + + let root = &tree[0]; + let mut best_ply = None; + let mut max_visits = 0; + + for &child_idx in &root.children { + let child = &tree[child_idx]; + if child.visits > max_visits { + max_visits = child.visits; + best_ply = child.ply_to_reach; + } + } + + best_ply + } + + fn take_stats(&mut self) -> AgentStats { + let stats = std::mem::take(&mut self.current_stats); + AgentStats::Mcts(stats) + } +} diff --git a/src/compute/src/agents/mcts/metrics.rs b/src/compute/src/agents/mcts/metrics.rs new file mode 100644 index 0000000..b747200 --- /dev/null +++ b/src/compute/src/agents/mcts/metrics.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +use crate::agents::metrics::CommonMetrics; + +#[derive(Debug, Serialize, Deserialize)] +pub struct MctsMetrics { + #[serde(flatten)] + pub common: CommonMetrics, + pub max_iterations: u32, + pub max_time_ms: Option, + pub exploration_constant: f64, + pub total_iterations: u64, + pub total_nodes_created: u64, +} diff --git a/src/compute/src/agents/mcts/node.rs b/src/compute/src/agents/mcts/node.rs new file mode 100644 index 0000000..02bfdaf --- /dev/null +++ b/src/compute/src/agents/mcts/node.rs @@ -0,0 +1,32 @@ +use crate::core::{Player, Ply}; + +#[derive(Debug, Default)] +pub struct MctsNode { + pub parent: Option, + pub children: Vec, + pub ply_to_reach: Option, + pub unexpanded_plies: Vec, + + pub visits: u32, + pub wins: f64, + + pub turn: Player, +} + +impl MctsNode { + pub fn new( + parent: Option, + ply_to_reach: Option, + unexpanded_plies: Vec, + turn: Player, + ) -> Self { + Self { + parent, + children: Vec::new(), + ply_to_reach, + unexpanded_plies, + turn, + ..Default::default() + } + } +} diff --git a/src/compute/src/agents/mcts/stats.rs b/src/compute/src/agents/mcts/stats.rs new file mode 100644 index 0000000..b933c50 --- /dev/null +++ b/src/compute/src/agents/mcts/stats.rs @@ -0,0 +1,27 @@ +#[derive(Debug, Default, Clone)] +pub struct MctsStats { + pub iterations: u32, + pub nodes_created: u32, + pub playout_steps: u64, +} + +impl MctsStats { + pub fn new() -> Self { + Self::default() + } +} + +#[derive(Debug, Default, Clone)] +pub struct MctsStatsAccumulator { + pub total_iterations: u64, + pub total_nodes_created: u64, + pub total_playout_steps: u64, +} + +impl MctsStatsAccumulator { + pub fn accumulate(&mut self, stats: &MctsStats) { + self.total_iterations += stats.iterations as u64; + self.total_nodes_created += stats.nodes_created as u64; + self.total_playout_steps += stats.playout_steps; + } +} diff --git a/src/compute/src/agents/metrics.rs b/src/compute/src/agents/metrics.rs index 432f82d..32a42ab 100644 --- a/src/compute/src/agents/metrics.rs +++ b/src/compute/src/agents/metrics.rs @@ -43,6 +43,7 @@ pub enum AgentType { #[default] Human, Minimax, + Mcts, } #[derive(Debug, Serialize, Deserialize)] @@ -85,3 +86,14 @@ pub struct MinimaxMetrics { pub defended_weight: i32, pub edge_penalty_weight: i32, } + +#[derive(Debug, Serialize, Deserialize)] +pub struct MctsMetrics { + #[serde(flatten)] + pub common: CommonMetrics, + pub max_iterations: u32, + pub max_time_ms: Option, + pub exploration_constant: f64, + pub total_iterations: u64, + pub total_nodes_created: u64, +} diff --git a/src/compute/src/agents/stats.rs b/src/compute/src/agents/stats.rs index 973d111..a11372c 100644 --- a/src/compute/src/agents/stats.rs +++ b/src/compute/src/agents/stats.rs @@ -1,11 +1,12 @@ use crate::agents::AgentStats; -use super::{AgentType, MinimaxStatsAccumulator}; +use super::{AgentType, MctsStatsAccumulator, MinimaxStatsAccumulator}; #[derive(Debug, Clone)] pub enum AgentStatsAccumulator { None, Minimax(MinimaxStatsAccumulator), + Mcts(MctsStatsAccumulator), } #[derive(Debug, Clone)] @@ -26,6 +27,7 @@ impl AgentRuntimeStats { AgentType::Minimax => { AgentStatsAccumulator::Minimax(MinimaxStatsAccumulator::default()) } + AgentType::Mcts => AgentStatsAccumulator::Mcts(MctsStatsAccumulator::default()), AgentType::Human => AgentStatsAccumulator::None, }, } @@ -40,6 +42,9 @@ impl AgentRuntimeStats { (AgentStatsAccumulator::Minimax(acc), AgentStats::Minimax(stats)) => { acc.accumulate(&stats); } + (AgentStatsAccumulator::Mcts(acc), AgentStats::Mcts(stats)) => { + acc.accumulate(&stats); + } (AgentStatsAccumulator::None, AgentStats::None) => { // Human agent doesn't provide stats } diff --git a/src/compute/src/bin/tournament.rs b/src/compute/src/bin/tournament.rs index af66c2b..0345d88 100644 --- a/src/compute/src/bin/tournament.rs +++ b/src/compute/src/bin/tournament.rs @@ -6,7 +6,7 @@ use compute::{ BreakthroughConfig, agents::{ AgentConfig, AgentRuntimeStats, AgentStatsAccumulator, AgentType, CommonMetrics, - HumanMetrics, MinimaxMetrics, append_record_to_jsonl, + HumanMetrics, MctsMetrics, MinimaxMetrics, append_record_to_jsonl, }, cli::build_command, core::{Board, BoardConfig, Player}, @@ -141,6 +141,25 @@ fn main() -> Result<()> { }; append_record_to_jsonl(&metrics, white_output).into_diagnostic()?; } + AgentStatsAccumulator::Mcts(acc) => { + let AgentConfig::Mcts { + max_iterations, + max_time_ms, + exploration_constant, + } = config.white_player.clone() + else { + unreachable!(); + }; + let metrics = MctsMetrics { + common: white_common, + max_iterations, + max_time_ms, + exploration_constant, + total_iterations: acc.total_iterations, + total_nodes_created: acc.total_nodes_created, + }; + append_record_to_jsonl(&metrics, white_output).into_diagnostic()?; + } AgentStatsAccumulator::None => { let metrics = HumanMetrics { common: white_common, @@ -188,6 +207,25 @@ fn main() -> Result<()> { }; append_record_to_jsonl(&metrics, black_output).into_diagnostic()?; } + AgentStatsAccumulator::Mcts(acc) => { + let AgentConfig::Mcts { + max_iterations, + max_time_ms, + exploration_constant, + } = config.black_player.clone() + else { + unreachable!(); + }; + let metrics = MctsMetrics { + common: black_common, + max_iterations, + max_time_ms, + exploration_constant, + total_iterations: acc.total_iterations, + total_nodes_created: acc.total_nodes_created, + }; + append_record_to_jsonl(&metrics, black_output).into_diagnostic()?; + } AgentStatsAccumulator::None => { let metrics = HumanMetrics { common: black_common, diff --git a/src/compute/src/gui/views/gameplay.rs b/src/compute/src/gui/views/gameplay.rs index 682e8b5..63f22cf 100644 --- a/src/compute/src/gui/views/gameplay.rs +++ b/src/compute/src/gui/views/gameplay.rs @@ -7,7 +7,7 @@ use crate::{ BreakthroughConfig, agents::{ AgentConfig, AgentStatsAccumulator, BreakthroughAgent, CommonMetrics, HumanMetrics, - MinimaxAgent, MinimaxMetrics, append_record_to_jsonl, + MctsAgent, MctsMetrics, MinimaxAgent, MinimaxMetrics, append_record_to_jsonl, }, core::{Board, BoardConfig, Player, Status}, gui::themes::BoardTheme, @@ -125,6 +125,9 @@ impl GameplayView { BreakthroughAgent::Minimax(minimax) => { Self::show_minimax_settings(ui, minimax, &opponent_player_label); } + BreakthroughAgent::Mcts(mcts) => { + Self::show_mcts_settings(ui, mcts, &opponent_player_label); + } BreakthroughAgent::Human => { ui.label(format!("Opponent ({opponent_player_label}) is also Human.",)); } @@ -168,6 +171,41 @@ impl GameplayView { ui.add(egui::Slider::new(&mut agent.max_depth, 1..=8).text("Search Depth")); } + fn show_mcts_settings(ui: &mut egui::Ui, agent: &mut MctsAgent, player_label: &str) { + ui.label(egui::RichText::new(format!("{player_label} Player (MCTS):")).strong()); + ui.add_space(5.0); + ui.horizontal(|ui| { + if ui + .radio(agent.max_time_ms.is_none(), "Iterations") + .clicked() + { + agent.max_time_ms = None; + } + if ui + .radio(agent.max_time_ms.is_some(), "Time Limit") + .clicked() + { + agent.max_time_ms = Some(1000); + } + }); + + if let Some(ref mut max_time) = agent.max_time_ms { + ui.add( + egui::Slider::new(max_time, 100..=10000) + .text("ms") + .logarithmic(true), + ); + } else { + ui.add( + egui::Slider::new(&mut agent.max_iterations, 1000..=100000) + .text("iters") + .logarithmic(true), + ); + } + + ui.add(egui::Slider::new(&mut agent.exploration_constant, 0.0..=5.0).text("exploration")); + } + fn show_agent_thinking(ui: &mut egui::Ui) { ui.horizontal(|ui| { ui.spinner(); @@ -524,6 +562,26 @@ impl GameplayView { }; let _ = append_record_to_jsonl(&metrics, self.white_output.clone()); } + AgentStatsAccumulator::Mcts(acc) => { + let AgentConfig::Mcts { + max_iterations, + max_time_ms, + exploration_constant, + .. + } = self.config.white_player + else { + unreachable!(); + }; + let metrics = MctsMetrics { + common: white_common, + max_iterations, + max_time_ms, + exploration_constant, + total_iterations: acc.total_iterations, + total_nodes_created: acc.total_nodes_created, + }; + let _ = append_record_to_jsonl(&metrics, self.white_output.clone()); + } AgentStatsAccumulator::None => { let metrics = HumanMetrics { common: white_common, @@ -571,6 +629,26 @@ impl GameplayView { }; let _ = append_record_to_jsonl(&metrics, self.black_output.clone()); } + AgentStatsAccumulator::Mcts(acc) => { + let AgentConfig::Mcts { + max_iterations, + max_time_ms, + exploration_constant, + .. + } = self.config.black_player + else { + unreachable!(); + }; + let metrics = MctsMetrics { + common: black_common, + max_iterations, + max_time_ms, + exploration_constant, + total_iterations: acc.total_iterations, + total_nodes_created: acc.total_nodes_created, + }; + let _ = append_record_to_jsonl(&metrics, self.black_output.clone()); + } AgentStatsAccumulator::None => { let metrics = HumanMetrics { common: black_common, diff --git a/src/compute/src/resources/configs/with_human/mcts_vs_human.toml b/src/compute/src/resources/configs/with_human/mcts_vs_human.toml new file mode 100644 index 0000000..3497e6e --- /dev/null +++ b/src/compute/src/resources/configs/with_human/mcts_vs_human.toml @@ -0,0 +1,9 @@ +board_width = 11 +board_height = 11 +seed = 42 + +[black_player] +type = "Mcts" + +[white_player] +type = "Human" diff --git a/src/compute/src/resources/configs/without_human/mcts_vs_minimax.toml b/src/compute/src/resources/configs/without_human/mcts_vs_minimax.toml new file mode 100644 index 0000000..156c7f3 --- /dev/null +++ b/src/compute/src/resources/configs/without_human/mcts_vs_minimax.toml @@ -0,0 +1,12 @@ +board_width = 8 +board_height = 8 +seed = 42 + +[black_player] +type = "Minimax" +max_depth = 5 + +[white_player] +type = "Mcts" +max_iterations = 75000 +exploration_constant = 1.41