Experiments H3

src/analysis/.gitignore

@@ -1 +1,3 @@
-*.jsonl
+*.jsonl
+*.toml
+*.png

src/analysis/experiments/h4/configs_generator.py

@@ -0,0 +1,47 @@
+import copy
+import tomli_w # type: ignore
+import os
+import shutil
+from pathlib import Path
+from itertools import product
+
+CONFIG_MCTS_VS_MINIMAX_BLACK = {
+    "board_width": 6,
+    "board_height": 7,
+    "white_player": {"type": "Mcts", "max_iterations": 5000},
+    "black_player": {"type": "Minimax"} # max_depth added in a loop
+}
+
+CONFIG_MCTS_VS_MINIMAX_WHITE = {
+    "board_width": 6,
+    "board_height": 7,
+    "white_player": {"type": "Minimax"}, # max_depth added in a loop
+    "black_player": {"type": "Mcts", "max_iterations": 5000}
+}
+
+def generate_mcts_vs_minimax_configs(output_dir: str):
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir)
+    os.makedirs(output_dir, exist_ok=True)
+
+    depths = [1, 2, 3, 4, 5, 6, 7]
+    colors = ["black", "white"] 
+
+    for depth, minimax_color in product(depths, colors):
+
+        if minimax_color == "black":
+            config = copy.deepcopy(CONFIG_MCTS_VS_MINIMAX_BLACK)
+            config["black_player"]["max_depth"] = depth
+        else:
+            config = copy.deepcopy(CONFIG_MCTS_VS_MINIMAX_WHITE)
+            config["white_player"]["max_depth"] = depth
+
+        filename = os.path.join(output_dir, f"minimax_{minimax_color}_depth_{depth}.toml")
+        with open(filename, "wb") as f:
+            tomli_w.dump(config, f)
+
+if __name__ == "__main__":
+    SCRIPT_DIR = Path(__file__).parent.resolve()
+    CONFIG_DIR = SCRIPT_DIR / "mcts"
+
+    generate_mcts_vs_minimax_configs(str(CONFIG_DIR))

src/analysis/experiments/h4/plots_generator.py

@@ -0,0 +1,162 @@
+import json
+import argparse
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from pathlib import Path
+
+def load_and_merge_data(white_filepath: Path, black_filepath: Path):
+    records = []
+
+    mcts_name = None
+    minimax_name = None
+
+    with open(white_filepath, 'r') as fw, open(black_filepath, 'r') as fb:
+        for line_white, line_black in zip(fw, fb):
+            if not line_white.strip() or not line_black.strip():
+                continue
+
+            data_white = json.loads(line_white)
+            data_black = json.loads(line_black)
+
+            if data_white['agent_type'] == 'Minimax':
+                minimax_data = data_white
+                mcts_data = data_black
+            else:
+                minimax_data = data_black
+                mcts_data = data_white
+
+            minimax_name = minimax_data['agent_type']
+            mcts_name = mcts_data['agent_type']
+
+            minimax_depth = minimax_data.get('max_depth')
+
+            records.append({
+                'minimax_depth': minimax_depth,
+                'mcts_color': mcts_data['agent_color'],
+                'minimax_color': minimax_data['agent_color'],
+                'mcts_won': int(mcts_data['agent_won']),
+                'minimax_won': int(minimax_data['agent_won']),
+                'total_moves': mcts_data['total_moves']
+            })
+
+    return pd.DataFrame(records), mcts_name, minimax_name
+
+def print_statistics(df: pd.DataFrame, mcts_name: str, minimax_name: str):
+    print(f"\n{'-'*75}")
+    print(f" TABLE 1: WIN RATE ({mcts_name} vs {minimax_name})")
+    print(f"{'-'*75}")
+
+    overall_win_stats = df.groupby('minimax_depth')['mcts_won'].mean() * 100
+    color_win_stats = df.groupby(['minimax_depth', 'mcts_color'])['mcts_won'].mean() * 100
+
+    col_w = f"{mcts_name} as White"
+    col_b = f"{mcts_name} as Black"
+
+    print(f"{'Depth':<8} | {col_w:<20} | {col_b:<20} | {'Overall Average':<18}")
+    print("-" * 75)
+
+    for depth in sorted(df['minimax_depth'].unique()):
+        win_white = color_win_stats.get((depth, 'White'), 0.0)
+        win_black = color_win_stats.get((depth, 'Black'), 0.0)
+        win_overall = overall_win_stats.get(depth, 0.0)
+
+        print(f"d={depth:<6} | {win_white:>19.1f}% | {win_black:>19.1f}% | {win_overall:>17.1f}%")
+
+    print(f"\n{'-'*80}")
+    print(f" TABLE 2: GAME LENGTH: Mean ± SD ({mcts_name} vs {minimax_name})")
+    print(f"{'-'*80}")
+
+    overall_moves = df.groupby('minimax_depth')['total_moves'].agg(['mean', 'std'])
+    color_moves = df.groupby(['minimax_depth', 'mcts_color'])['total_moves'].agg(['mean', 'std'])
+
+    print(f"{'Depth':<8} | {col_w:<22} | {col_b:<22} | {'Overall Average':<20}")
+    print("-" * 80)
+
+    def format_moves(stats_df, key):
+        try:
+            row = stats_df.loc[key]
+            std_val = row['std']
+            if pd.isna(std_val):
+                std_val = 0.0
+            return f"{row['mean']:.1f} ± {std_val:.1f}"
+        except KeyError:
+            return "N/A"
+
+    for depth in sorted(df['minimax_depth'].unique()):
+        moves_white = format_moves(color_moves, (depth, 'White'))
+        moves_black = format_moves(color_moves, (depth, 'Black'))
+        moves_overall = format_moves(overall_moves, depth)
+
+        print(f"d={depth:<6} | {moves_white:>22} | {moves_black:>22} | {moves_overall:>19}")
+
+def generate_plots(df: pd.DataFrame, mcts_name: str):
+    sns.set_theme(style="whitegrid")
+
+    mcts_stats = df.groupby(['minimax_depth', 'mcts_color'])['mcts_won'].mean().reset_index()
+    mcts_stats.rename(columns={'mcts_won': 'win_rate'}, inplace=True)
+
+    mcts_stats['max_win'] = 1.0
+
+    plt.figure(figsize=(8, 6))
+
+    sns.barplot(
+        data=mcts_stats, x='minimax_depth', y='max_win', hue='mcts_color',
+        palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True, alpha=0.3,
+        legend=False
+    )
+
+    sns.barplot(
+        data=mcts_stats, x='minimax_depth', y='win_rate', hue='mcts_color',
+        palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True
+    )
+
+    plt.xlabel('Głębokość przeszukiwania drzewa gry', fontsize=12)
+    plt.ylabel('Współczynnik zwycięstw', fontsize=12)
+    plt.ylim(0, 1.05)
+    plt.legend(title=f'Kolor {mcts_name}')
+    plt.tight_layout()
+    mcts_filename = f"{mcts_name.lower()}_win_rate.png"
+    plt.savefig(mcts_filename, dpi=300)
+    plt.close()
+
+    plt.figure(figsize=(8, 6))
+
+    sns.boxplot(
+        data=df, x='minimax_depth', y='total_moves', hue='mcts_color',
+        palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True,
+        linewidth=1.5, fliersize=4
+    )
+
+    plt.xlabel('Głębokość przeszukiwania drzewa gry', fontsize=12)
+    plt.ylabel('Całkowita liczba ruchów', fontsize=12)
+    plt.legend(title=f'Kolor {mcts_name}')
+    plt.tight_layout()
+
+    moves_filename = f"{mcts_name.lower()}_total_moves.png"
+    plt.savefig(moves_filename, dpi=300)
+    plt.close()
+
+    print(f"Plots saved:")
+    print(f" -> {mcts_filename}")
+    print(f" -> {moves_filename}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate plots from game results.")
+    parser.add_argument("--white", type=Path, required=True, help="Path to the white player JSONL file")
+    parser.add_argument("--black", type=Path, required=True, help="Path to the black player JSONL file")
+
+    args = parser.parse_args()
+
+    if not args.white.exists() or not args.black.exists():
+        print("Error: Input file(s) not found.")
+        return
+
+    df, mcts_name, minimax_name = load_and_merge_data(args.white, args.black)
+
+    print(f"Processed {len(df)} games.")
+    print_statistics(df, mcts_name, minimax_name)
+    generate_plots(df, mcts_name)
+
+if __name__ == "__main__":
+    main()

src/analysis/pyproject.toml

@@ -6,6 +6,9 @@ requires-python = ">=3.13"
 dependencies = [
     "matplotlib>=3.10.9",
     "numpy>=2.4.4",
+    "pandas>=3.0.3",
+    "seaborn>=0.13.2",
+    "tomli-w>=1.2.0",
 ]
 
 [dependency-groups]

src/analysis/runner.py

@@ -3,68 +3,78 @@
 import sys
 from pathlib import Path
 
-def main():
-    parser = argparse.ArgumentParser(description="Automate running experiments for the Breakthrough game.")
+def run_experiments(config_dir, runs, bin_path, base_seed=42, 
+                    white_output=None, black_output=None, 
+                    board_width=None, board_height=None):
+    config_dir = Path(config_dir)
 
-    parser.add_argument("config_dir", type=Path, help="Directory containing .toml configuration files")
-    parser.add_argument("--runs", type=int, default=10, help="Number of runs per configuration file")
-    parser.add_argument("--bin", type=str, help="Path to the Rust executable")
-    parser.add_argument("--base-seed", type=int, default=42, help="Initial seed for the first run")
-
-    parser.add_argument("--white-output", type=str, 
-                        help="Path to the output JSONL file for the white player. Supports {config} and {seed} formatting.")
-    parser.add_argument("--black-output", type=str, 
-                        help="Path to the output JSONL file for the black player. Supports {config} and {seed} formatting.")
-    parser.add_argument("--board-width", type=int, help="Width of the game board")
-    parser.add_argument("--board-height", type=int, help="Height of the game board")
-
-    args = parser.parse_args()
-
-    if not args.config_dir.is_dir():
-        print(f"Error: Directory '{args.config_dir}' does not exist.")
+    if not config_dir.is_dir():
+        print(f"Error: Directory '{config_dir}' does not exist.")
         sys.exit(1)
 
-    toml_files = list(args.config_dir.glob("*.toml"))
+    toml_files = list(config_dir.glob("*.toml"))
     if not toml_files:
-        print(f"No .toml files found in directory '{args.config_dir}'.")
+        print(f"No .toml files found in directory '{config_dir}'.")
         sys.exit(0)
 
-    print(f"Found {len(toml_files)} configuration files. Each will be run {args.runs} times.")
+    print(f"Found {len(toml_files)} configuration files. Each will be run {runs} times.")
 
     for toml_file in toml_files:
         print(f"\n{'-'*40}\nProcessing configuration: {toml_file.name}\n{'-'*40}")
 
-        for i in range(args.runs):
-            current_seed = args.base_seed + i
-
-            cmd = [args.bin, str(toml_file)]
+        for i in range(runs):
+            current_seed = base_seed + i
+            cmd = [bin_path, str(toml_file)]
 
-            if args.white_output:
-                white_out = args.white_output.format(config=toml_file.stem, seed=current_seed)
+            if white_output:
+                white_out = white_output.format(config=toml_file.stem, seed=current_seed)
                 cmd.extend(["--white-output", white_out])
 
-            if args.black_output:
-                black_out = args.black_output.format(config=toml_file.stem, seed=current_seed)
+            if black_output:
+                black_out = black_output.format(config=toml_file.stem, seed=current_seed)
                 cmd.extend(["--black-output", black_out])
 
-            if args.board_width is not None:
-                cmd.extend(["--board-width", str(args.board_width)])
+            if board_width is not None:
+                cmd.extend(["--board-width", str(board_width)])
 
-            if args.board_height is not None:
-                cmd.extend(["--board-height", str(args.board_height)])
+            if board_height is not None:
+                cmd.extend(["--board-height", str(board_height)])
 
             cmd.extend(["--seed", str(current_seed)])
 
-            print(f"Run {i+1}/{args.runs} | Seed: {current_seed}")
-            print(f"Command: {' '.join(cmd)}")
-
+            print(f"Run {i+1}/{runs} | Seed: {current_seed}")
+
             try:
                 subprocess.run(cmd, check=True)
             except subprocess.CalledProcessError as e:
                 print(f"Error running iteration {i+1} for {toml_file.name}: Exited with code {e.returncode}")
             except FileNotFoundError:
-                print(f"Error: Executable '{args.bin}' not found. Make sure the project is built (e.g., cargo build --release).")
+                print(f"Error: Executable '{bin_path}' not found.")
                 sys.exit(1)
 
+def main():
+    parser = argparse.ArgumentParser(description="Automate running experiments for the Breakthrough game.")
+    parser.add_argument("config_dir", type=Path, help="Directory containing .toml configuration files")
+    parser.add_argument("--runs", type=int, default=10, help="Number of runs per configuration file")
+    parser.add_argument("--bin", type=str, required=True, help="Path to the Rust executable")
+    parser.add_argument("--base-seed", type=int, default=42, help="Initial seed for the first run")
+    parser.add_argument("--white-output", type=str)
+    parser.add_argument("--black-output", type=str)
+    parser.add_argument("--board-width", type=int)
+    parser.add_argument("--board-height", type=int)
+
+    args = parser.parse_args()
+
+    run_experiments(
+        config_dir=args.config_dir,
+        runs=args.runs,
+        bin_path=args.bin,
+        base_seed=args.base_seed,
+        white_output=args.white_output,
+        black_output=args.black_output,
+        board_width=args.board_width,
+        board_height=args.board_height
+    )
+
 if __name__ == "__main__":
     main()