From f9b400bb7d31a20baea78e0f1ea5fc0d63299863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Gr=C4=85cikowski?= <01180781@pw.edu.pl> Date: Sun, 17 May 2026 15:22:05 +0200 Subject: [PATCH 1/2] add h4 generator and plots --- src/analysis/.gitignore | 4 +- .../experiments/h4/configs_generator.py | 47 +++++ .../experiments/h4/plots_generator.py | 162 ++++++++++++++++++ src/analysis/pyproject.toml | 3 + src/analysis/runner.py | 84 +++++---- src/analysis/uv.lock | 90 ++++++++++ src/compute/src/agents/metrics.rs | 4 +- src/compute/src/agents/stats.rs | 6 +- src/compute/src/bin/tournament.rs | 8 +- src/compute/src/gui/views/gameplay.rs | 8 +- 10 files changed, 365 insertions(+), 51 deletions(-) create mode 100644 src/analysis/experiments/h4/configs_generator.py create mode 100644 src/analysis/experiments/h4/plots_generator.py diff --git a/src/analysis/.gitignore b/src/analysis/.gitignore index 39646a4..6e23b8e 100644 --- a/src/analysis/.gitignore +++ b/src/analysis/.gitignore @@ -1 +1,3 @@ -*.jsonl \ No newline at end of file +*.jsonl +*.toml +*.png \ No newline at end of file diff --git a/src/analysis/experiments/h4/configs_generator.py b/src/analysis/experiments/h4/configs_generator.py new file mode 100644 index 0000000..0c178d1 --- /dev/null +++ b/src/analysis/experiments/h4/configs_generator.py @@ -0,0 +1,47 @@ +import copy +import tomli_w # type: ignore +import os +import shutil +from pathlib import Path +from itertools import product + +CONFIG_MCTS_VS_MINIMAX_BLACK = { + "board_width": 6, + "board_height": 7, + "white_player": {"type": "Mcts", "max_iterations": 5000}, + "black_player": {"type": "Minimax"} # max_depth added in a loop +} + +CONFIG_MCTS_VS_MINIMAX_WHITE = { + "board_width": 6, + "board_height": 7, + "white_player": {"type": "Minimax"}, # max_depth added in a loop + "black_player": {"type": "Mcts", "max_iterations": 5000} +} + +def generate_mcts_vs_minimax_configs(output_dir: str): + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir, exist_ok=True) + + depths = [1, 2, 3, 4, 5, 6, 7] + colors = ["black", "white"] + + for depth, minimax_color in product(depths, colors): + + if minimax_color == "black": + config = copy.deepcopy(CONFIG_MCTS_VS_MINIMAX_BLACK) + config["black_player"]["max_depth"] = depth + else: + config = copy.deepcopy(CONFIG_MCTS_VS_MINIMAX_WHITE) + config["white_player"]["max_depth"] = depth + + filename = os.path.join(output_dir, f"minimax_{minimax_color}_depth_{depth}.toml") + with open(filename, "wb") as f: + tomli_w.dump(config, f) + +if __name__ == "__main__": + SCRIPT_DIR = Path(__file__).parent.resolve() + CONFIG_DIR = SCRIPT_DIR / "mcts" + + generate_mcts_vs_minimax_configs(str(CONFIG_DIR)) \ No newline at end of file diff --git a/src/analysis/experiments/h4/plots_generator.py b/src/analysis/experiments/h4/plots_generator.py new file mode 100644 index 0000000..13524b8 --- /dev/null +++ b/src/analysis/experiments/h4/plots_generator.py @@ -0,0 +1,162 @@ +import json +import argparse +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +from pathlib import Path + +def load_and_merge_data(white_filepath: Path, black_filepath: Path): + records = [] + + mcts_name = None + minimax_name = None + + with open(white_filepath, 'r') as fw, open(black_filepath, 'r') as fb: + for line_white, line_black in zip(fw, fb): + if not line_white.strip() or not line_black.strip(): + continue + + data_white = json.loads(line_white) + data_black = json.loads(line_black) + + if data_white['agent_type'] == 'Minimax': + minimax_data = data_white + mcts_data = data_black + else: + minimax_data = data_black + mcts_data = data_white + + minimax_name = minimax_data['agent_type'] + mcts_name = mcts_data['agent_type'] + + minimax_depth = minimax_data.get('max_depth') + + records.append({ + 'minimax_depth': minimax_depth, + 'mcts_color': mcts_data['agent_color'], + 'minimax_color': minimax_data['agent_color'], + 'mcts_won': int(mcts_data['agent_won']), + 'minimax_won': int(minimax_data['agent_won']), + 'total_moves': mcts_data['total_moves'] + }) + + return pd.DataFrame(records), mcts_name, minimax_name + +def print_statistics(df: pd.DataFrame, mcts_name: str, minimax_name: str): + print(f"\n{'-'*75}") + print(f" TABLE 1: WIN RATE ({mcts_name} vs {minimax_name})") + print(f"{'-'*75}") + + overall_win_stats = df.groupby('minimax_depth')['mcts_won'].mean() * 100 + color_win_stats = df.groupby(['minimax_depth', 'mcts_color'])['mcts_won'].mean() * 100 + + col_w = f"{mcts_name} as White" + col_b = f"{mcts_name} as Black" + + print(f"{'Depth':<8} | {col_w:<20} | {col_b:<20} | {'Overall Average':<18}") + print("-" * 75) + + for depth in sorted(df['minimax_depth'].unique()): + win_white = color_win_stats.get((depth, 'White'), 0.0) + win_black = color_win_stats.get((depth, 'Black'), 0.0) + win_overall = overall_win_stats.get(depth, 0.0) + + print(f"d={depth:<6} | {win_white:>19.1f}% | {win_black:>19.1f}% | {win_overall:>17.1f}%") + + print(f"\n{'-'*80}") + print(f" TABLE 2: GAME LENGTH: Mean ± SD ({mcts_name} vs {minimax_name})") + print(f"{'-'*80}") + + overall_moves = df.groupby('minimax_depth')['total_moves'].agg(['mean', 'std']) + color_moves = df.groupby(['minimax_depth', 'mcts_color'])['total_moves'].agg(['mean', 'std']) + + print(f"{'Depth':<8} | {col_w:<22} | {col_b:<22} | {'Overall Average':<20}") + print("-" * 80) + + def format_moves(stats_df, key): + try: + row = stats_df.loc[key] + std_val = row['std'] + if pd.isna(std_val): + std_val = 0.0 + return f"{row['mean']:.1f} ± {std_val:.1f}" + except KeyError: + return "N/A" + + for depth in sorted(df['minimax_depth'].unique()): + moves_white = format_moves(color_moves, (depth, 'White')) + moves_black = format_moves(color_moves, (depth, 'Black')) + moves_overall = format_moves(overall_moves, depth) + + print(f"d={depth:<6} | {moves_white:>22} | {moves_black:>22} | {moves_overall:>19}") + +def generate_plots(df: pd.DataFrame, mcts_name: str): + sns.set_theme(style="whitegrid") + + mcts_stats = df.groupby(['minimax_depth', 'mcts_color'])['mcts_won'].mean().reset_index() + mcts_stats.rename(columns={'mcts_won': 'win_rate'}, inplace=True) + + mcts_stats['max_win'] = 1.0 + + plt.figure(figsize=(8, 6)) + + sns.barplot( + data=mcts_stats, x='minimax_depth', y='max_win', hue='mcts_color', + palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True, alpha=0.3, + legend=False + ) + + sns.barplot( + data=mcts_stats, x='minimax_depth', y='win_rate', hue='mcts_color', + palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True + ) + + plt.xlabel('Głębokość przeszukiwania drzewa gry', fontsize=12) + plt.ylabel('Współczynnik zwycięstw', fontsize=12) + plt.ylim(0, 1.05) + plt.legend(title=f'Kolor {mcts_name}') + plt.tight_layout() + mcts_filename = f"{mcts_name.lower()}_win_rate.png" + plt.savefig(mcts_filename, dpi=300) + plt.close() + + plt.figure(figsize=(8, 6)) + + sns.boxplot( + data=df, x='minimax_depth', y='total_moves', hue='mcts_color', + palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True, + linewidth=1.5, fliersize=4 + ) + + plt.xlabel('Głębokość przeszukiwania drzewa gry', fontsize=12) + plt.ylabel('Całkowita liczba ruchów', fontsize=12) + plt.legend(title=f'Kolor {mcts_name}') + plt.tight_layout() + + moves_filename = f"{mcts_name.lower()}_total_moves.png" + plt.savefig(moves_filename, dpi=300) + plt.close() + + print(f"Plots saved:") + print(f" -> {mcts_filename}") + print(f" -> {moves_filename}") + +def main(): + parser = argparse.ArgumentParser(description="Generate plots from game results.") + parser.add_argument("--white", type=Path, required=True, help="Path to the white player JSONL file") + parser.add_argument("--black", type=Path, required=True, help="Path to the black player JSONL file") + + args = parser.parse_args() + + if not args.white.exists() or not args.black.exists(): + print("Error: Input file(s) not found.") + return + + df, mcts_name, minimax_name = load_and_merge_data(args.white, args.black) + + print(f"Processed {len(df)} games.") + print_statistics(df, mcts_name, minimax_name) + generate_plots(df, mcts_name) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/analysis/pyproject.toml b/src/analysis/pyproject.toml index 324f8ab..51129c6 100644 --- a/src/analysis/pyproject.toml +++ b/src/analysis/pyproject.toml @@ -6,6 +6,9 @@ requires-python = ">=3.13" dependencies = [ "matplotlib>=3.10.9", "numpy>=2.4.4", + "pandas>=3.0.3", + "seaborn>=0.13.2", + "tomli-w>=1.2.0", ] [dependency-groups] diff --git a/src/analysis/runner.py b/src/analysis/runner.py index b82de65..8625069 100644 --- a/src/analysis/runner.py +++ b/src/analysis/runner.py @@ -3,68 +3,78 @@ import sys from pathlib import Path -def main(): - parser = argparse.ArgumentParser(description="Automate running experiments for the Breakthrough game.") +def run_experiments(config_dir, runs, bin_path, base_seed=42, + white_output=None, black_output=None, + board_width=None, board_height=None): + config_dir = Path(config_dir) - parser.add_argument("config_dir", type=Path, help="Directory containing .toml configuration files") - parser.add_argument("--runs", type=int, default=10, help="Number of runs per configuration file") - parser.add_argument("--bin", type=str, help="Path to the Rust executable") - parser.add_argument("--base-seed", type=int, default=42, help="Initial seed for the first run") - - parser.add_argument("--white-output", type=str, - help="Path to the output JSONL file for the white player. Supports {config} and {seed} formatting.") - parser.add_argument("--black-output", type=str, - help="Path to the output JSONL file for the black player. Supports {config} and {seed} formatting.") - parser.add_argument("--board-width", type=int, help="Width of the game board") - parser.add_argument("--board-height", type=int, help="Height of the game board") - - args = parser.parse_args() - - if not args.config_dir.is_dir(): - print(f"Error: Directory '{args.config_dir}' does not exist.") + if not config_dir.is_dir(): + print(f"Error: Directory '{config_dir}' does not exist.") sys.exit(1) - toml_files = list(args.config_dir.glob("*.toml")) + toml_files = list(config_dir.glob("*.toml")) if not toml_files: - print(f"No .toml files found in directory '{args.config_dir}'.") + print(f"No .toml files found in directory '{config_dir}'.") sys.exit(0) - print(f"Found {len(toml_files)} configuration files. Each will be run {args.runs} times.") + print(f"Found {len(toml_files)} configuration files. Each will be run {runs} times.") for toml_file in toml_files: print(f"\n{'-'*40}\nProcessing configuration: {toml_file.name}\n{'-'*40}") - for i in range(args.runs): - current_seed = args.base_seed + i - - cmd = [args.bin, str(toml_file)] + for i in range(runs): + current_seed = base_seed + i + cmd = [bin_path, str(toml_file)] - if args.white_output: - white_out = args.white_output.format(config=toml_file.stem, seed=current_seed) + if white_output: + white_out = white_output.format(config=toml_file.stem, seed=current_seed) cmd.extend(["--white-output", white_out]) - if args.black_output: - black_out = args.black_output.format(config=toml_file.stem, seed=current_seed) + if black_output: + black_out = black_output.format(config=toml_file.stem, seed=current_seed) cmd.extend(["--black-output", black_out]) - if args.board_width is not None: - cmd.extend(["--board-width", str(args.board_width)]) + if board_width is not None: + cmd.extend(["--board-width", str(board_width)]) - if args.board_height is not None: - cmd.extend(["--board-height", str(args.board_height)]) + if board_height is not None: + cmd.extend(["--board-height", str(board_height)]) cmd.extend(["--seed", str(current_seed)]) - print(f"Run {i+1}/{args.runs} | Seed: {current_seed}") - print(f"Command: {' '.join(cmd)}") - + print(f"Run {i+1}/{runs} | Seed: {current_seed}") + try: subprocess.run(cmd, check=True) except subprocess.CalledProcessError as e: print(f"Error running iteration {i+1} for {toml_file.name}: Exited with code {e.returncode}") except FileNotFoundError: - print(f"Error: Executable '{args.bin}' not found. Make sure the project is built (e.g., cargo build --release).") + print(f"Error: Executable '{bin_path}' not found.") sys.exit(1) +def main(): + parser = argparse.ArgumentParser(description="Automate running experiments for the Breakthrough game.") + parser.add_argument("config_dir", type=Path, help="Directory containing .toml configuration files") + parser.add_argument("--runs", type=int, default=10, help="Number of runs per configuration file") + parser.add_argument("--bin", type=str, required=True, help="Path to the Rust executable") + parser.add_argument("--base-seed", type=int, default=42, help="Initial seed for the first run") + parser.add_argument("--white-output", type=str) + parser.add_argument("--black-output", type=str) + parser.add_argument("--board-width", type=int) + parser.add_argument("--board-height", type=int) + + args = parser.parse_args() + + run_experiments( + config_dir=args.config_dir, + runs=args.runs, + bin_path=args.bin, + base_seed=args.base_seed, + white_output=args.white_output, + black_output=args.black_output, + board_width=args.board_width, + board_height=args.board_height + ) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/analysis/uv.lock b/src/analysis/uv.lock index 68e7e06..c8ff8fc 100644 --- a/src/analysis/uv.lock +++ b/src/analysis/uv.lock @@ -1,6 +1,14 @@ version = 1 revision = 3 requires-python = ">=3.13" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] [[package]] name = "analysis" @@ -9,6 +17,9 @@ source = { virtual = "." } dependencies = [ { name = "matplotlib" }, { name = "numpy" }, + { name = "pandas" }, + { name = "seaborn" }, + { name = "tomli-w" }, ] [package.dev-dependencies] @@ -21,6 +32,9 @@ dev = [ requires-dist = [ { name = "matplotlib", specifier = ">=3.10.9" }, { name = "numpy", specifier = ">=2.4.4" }, + { name = "pandas", specifier = ">=3.0.3" }, + { name = "seaborn", specifier = ">=0.13.2" }, + { name = "tomli-w", specifier = ">=1.2.0" }, ] [package.metadata.requires-dev] @@ -317,6 +331,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, +] + [[package]] name = "pillow" version = "12.2.0" @@ -455,6 +513,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] +[[package]] +name = "seaborn" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -463,3 +535,21 @@ sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68 wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] + +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] diff --git a/src/compute/src/agents/metrics.rs b/src/compute/src/agents/metrics.rs index 32a42ab..fda6f97 100644 --- a/src/compute/src/agents/metrics.rs +++ b/src/compute/src/agents/metrics.rs @@ -55,9 +55,9 @@ pub struct CommonMetrics { pub agent_color: Player, pub agent_won: bool, pub pieces_remaining: u8, - pub total_time_ms: u128, + pub move_times_ms: Vec, pub total_moves: usize, - pub moves: String, + pub moves: Vec, pub opponent_type: AgentType, pub opponent_pieces_remaining: u8, } diff --git a/src/compute/src/agents/stats.rs b/src/compute/src/agents/stats.rs index a11372c..e7ac208 100644 --- a/src/compute/src/agents/stats.rs +++ b/src/compute/src/agents/stats.rs @@ -11,7 +11,7 @@ pub enum AgentStatsAccumulator { #[derive(Debug, Clone)] pub struct AgentRuntimeStats { - pub total_time_ms: u128, + pub move_times_ms: Vec, pub total_moves: usize, pub move_history: Vec, pub stats_accumulator: AgentStatsAccumulator, @@ -20,7 +20,7 @@ pub struct AgentRuntimeStats { impl AgentRuntimeStats { pub fn new(agent_type: &AgentType) -> Self { Self { - total_time_ms: 0, + move_times_ms: vec![], total_moves: 0, move_history: Vec::new(), stats_accumulator: match agent_type { @@ -36,7 +36,7 @@ impl AgentRuntimeStats { pub fn record_move(&mut self, ply_str: String, time_ms: u128, move_stats: AgentStats) { self.total_moves += 1; self.move_history.push(ply_str); - self.total_time_ms += time_ms; + self.move_times_ms.push(time_ms); match (&mut self.stats_accumulator, move_stats) { (AgentStatsAccumulator::Minimax(acc), AgentStats::Minimax(stats)) => { diff --git a/src/compute/src/bin/tournament.rs b/src/compute/src/bin/tournament.rs index 0345d88..98e702c 100644 --- a/src/compute/src/bin/tournament.rs +++ b/src/compute/src/bin/tournament.rs @@ -110,9 +110,9 @@ fn main() -> Result<()> { agent_color: Player::White, agent_won: is_white_won, pieces_remaining: white_pieces, - total_time_ms: white_stats.total_time_ms, + move_times_ms: white_stats.move_times_ms, total_moves: white_stats.total_moves, - moves: white_stats.move_history.join(";"), + moves: white_stats.move_history, opponent_type: (&config.black_player).into(), opponent_pieces_remaining: black_pieces, }; @@ -176,9 +176,9 @@ fn main() -> Result<()> { agent_color: Player::Black, agent_won: !is_white_won, pieces_remaining: black_pieces, - total_time_ms: black_stats.total_time_ms, + move_times_ms: black_stats.move_times_ms, total_moves: black_stats.total_moves, - moves: black_stats.move_history.join(";"), + moves: black_stats.move_history, opponent_type: (&config.white_player).into(), opponent_pieces_remaining: white_pieces, }; diff --git a/src/compute/src/gui/views/gameplay.rs b/src/compute/src/gui/views/gameplay.rs index 63f22cf..6747368 100644 --- a/src/compute/src/gui/views/gameplay.rs +++ b/src/compute/src/gui/views/gameplay.rs @@ -531,9 +531,9 @@ impl GameplayView { agent_color: Player::White, agent_won: is_white_won, pieces_remaining: white_pieces, - total_time_ms: self.gameplay_controller.white_stats.total_time_ms, + move_times_ms: self.gameplay_controller.white_stats.move_times_ms.clone(), total_moves: self.gameplay_controller.white_stats.total_moves, - moves: self.gameplay_controller.white_stats.move_history.join(";"), + moves: self.gameplay_controller.white_stats.move_history.clone(), opponent_type: (&self.config.black_player).into(), opponent_pieces_remaining: black_pieces, }; @@ -598,9 +598,9 @@ impl GameplayView { agent_color: Player::Black, agent_won: !is_white_won, pieces_remaining: black_pieces, - total_time_ms: self.gameplay_controller.black_stats.total_time_ms, + move_times_ms: self.gameplay_controller.black_stats.move_times_ms.clone(), total_moves: self.gameplay_controller.black_stats.total_moves, - moves: self.gameplay_controller.black_stats.move_history.join(";"), + moves: self.gameplay_controller.black_stats.move_history.clone(), opponent_type: (&self.config.white_player).into(), opponent_pieces_remaining: white_pieces, }; From 4f5c7dc0d96578f431a7baf3e2fa89eddd11640a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Gr=C4=85cikowski?= <01180781@pw.edu.pl> Date: Tue, 19 May 2026 15:18:03 +0200 Subject: [PATCH 2/2] add average move time plot --- .../experiments/h4/configs_generator.py | 4 +- .../experiments/h4/plots_generator.py | 114 ++++++++++++++---- 2 files changed, 91 insertions(+), 27 deletions(-) diff --git a/src/analysis/experiments/h4/configs_generator.py b/src/analysis/experiments/h4/configs_generator.py index 0c178d1..f375eb3 100644 --- a/src/analysis/experiments/h4/configs_generator.py +++ b/src/analysis/experiments/h4/configs_generator.py @@ -8,7 +8,7 @@ CONFIG_MCTS_VS_MINIMAX_BLACK = { "board_width": 6, "board_height": 7, - "white_player": {"type": "Mcts", "max_iterations": 5000}, + "white_player": {"type": "Mcts", "max_iterations": 10000}, "black_player": {"type": "Minimax"} # max_depth added in a loop } @@ -16,7 +16,7 @@ "board_width": 6, "board_height": 7, "white_player": {"type": "Minimax"}, # max_depth added in a loop - "black_player": {"type": "Mcts", "max_iterations": 5000} + "black_player": {"type": "Mcts", "max_iterations": 10000} } def generate_mcts_vs_minimax_configs(output_dir: str): diff --git a/src/analysis/experiments/h4/plots_generator.py b/src/analysis/experiments/h4/plots_generator.py index 13524b8..3686cdb 100644 --- a/src/analysis/experiments/h4/plots_generator.py +++ b/src/analysis/experiments/h4/plots_generator.py @@ -5,14 +5,15 @@ import seaborn as sns from pathlib import Path -def load_and_merge_data(white_filepath: Path, black_filepath: Path): - records = [] +def load_all_data(white_filepath: Path, black_filepath: Path): + game_records = [] + move_records = [] mcts_name = None minimax_name = None with open(white_filepath, 'r') as fw, open(black_filepath, 'r') as fb: - for line_white, line_black in zip(fw, fb): + for line_num, (line_white, line_black) in enumerate(zip(fw, fb), start=1): if not line_white.strip() or not line_black.strip(): continue @@ -20,27 +21,45 @@ def load_and_merge_data(white_filepath: Path, black_filepath: Path): data_black = json.loads(line_black) if data_white['agent_type'] == 'Minimax': - minimax_data = data_white - mcts_data = data_black + minimax_data, minimax_color = data_white, 'White' + mcts_data, mcts_color = data_black, 'Black' else: - minimax_data = data_black - mcts_data = data_white + minimax_data, minimax_color = data_black, 'Black' + mcts_data, mcts_color = data_white, 'White' minimax_name = minimax_data['agent_type'] mcts_name = mcts_data['agent_type'] + depth = minimax_data.get('max_depth') - minimax_depth = minimax_data.get('max_depth') - - records.append({ - 'minimax_depth': minimax_depth, - 'mcts_color': mcts_data['agent_color'], - 'minimax_color': minimax_data['agent_color'], + game_records.append({ + 'minimax_depth': depth, + 'mcts_color': mcts_color, + 'minimax_color': minimax_color, 'mcts_won': int(mcts_data['agent_won']), 'minimax_won': int(minimax_data['agent_won']), 'total_moves': mcts_data['total_moves'] }) - return pd.DataFrame(records), mcts_name, minimax_name + for i, time_ms in enumerate(mcts_data['move_times_ms']): + move_records.append({ + 'Category': mcts_name, + 'Agent Move Number': i + 1, + 'Thinking Time (ms)': time_ms, + 'Sort_Key': 0 + }) + + for i, time_ms in enumerate(minimax_data['move_times_ms']): + move_records.append({ + 'Category': f'{minimax_name} (głębokość = {depth})', + 'Agent Move Number': i + 1, + 'Thinking Time (ms)': time_ms, + 'Sort_Key': depth + }) + + df_games = pd.DataFrame(game_records) + df_moves = pd.DataFrame(move_records) + + return df_games, df_moves, mcts_name, minimax_name def print_statistics(df: pd.DataFrame, mcts_name: str, minimax_name: str): print(f"\n{'-'*75}") @@ -90,10 +109,52 @@ def format_moves(stats_df, key): print(f"d={depth:<6} | {moves_white:>22} | {moves_black:>22} | {moves_overall:>19}") -def generate_plots(df: pd.DataFrame, mcts_name: str): +def generate_move_time_plots(df_moves: pd.DataFrame, mcts_name: str, minimax_name: str): + if df_moves.empty: + print("Error: No move data found.") + return + sns.set_theme(style="whitegrid") + df_moves = df_moves.sort_values(by=['Sort_Key']) + + plt.figure(figsize=(12, 7)) + + depths = sorted([d for d in df_moves['Sort_Key'].unique() if d > 0]) + minimax_colors = sns.color_palette("Reds", n_colors=len(depths) + 2).as_hex()[2:] + + palette = {f'{mcts_name}': '#1f77b4'} + for depth, color in zip(depths, minimax_colors): + palette[f'{minimax_name} (głębokość = {depth})'] = color + + sns.lineplot( + data=df_moves, + x='Agent Move Number', + y='Thinking Time (ms)', + hue='Category', + palette=palette, + marker='o', + markersize=5, + linewidth=2, + errorbar=('ci', 95) + ) + + plt.xlabel('Numer ruchu', fontsize=12) + plt.ylabel('Czas namysłu (ms)', fontsize=12) + + plt.xticks(sorted(df_moves['Agent Move Number'].unique())) + plt.legend(title='Algorytm') + plt.tight_layout() + + time_filename = f"{mcts_name.lower()}_thinking_time.png" + plt.savefig(time_filename, dpi=300) + plt.close() + + print(f" -> {time_filename}") - mcts_stats = df.groupby(['minimax_depth', 'mcts_color'])['mcts_won'].mean().reset_index() +def generate_plots(df_games: pd.DataFrame, mcts_name: str): + sns.set_theme(style="whitegrid") + + mcts_stats = df_games.groupby(['minimax_depth', 'mcts_color'])['mcts_won'].mean().reset_index() mcts_stats.rename(columns={'mcts_won': 'win_rate'}, inplace=True) mcts_stats['max_win'] = 1.0 @@ -103,12 +164,13 @@ def generate_plots(df: pd.DataFrame, mcts_name: str): sns.barplot( data=mcts_stats, x='minimax_depth', y='max_win', hue='mcts_color', palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True, alpha=0.3, - legend=False + legend=False, hue_order=['White', 'Black'] ) sns.barplot( data=mcts_stats, x='minimax_depth', y='win_rate', hue='mcts_color', - palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True + palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True, + hue_order=['White', 'Black'] ) plt.xlabel('Głębokość przeszukiwania drzewa gry', fontsize=12) @@ -123,9 +185,9 @@ def generate_plots(df: pd.DataFrame, mcts_name: str): plt.figure(figsize=(8, 6)) sns.boxplot( - data=df, x='minimax_depth', y='total_moves', hue='mcts_color', + data=df_games, x='minimax_depth', y='total_moves', hue='mcts_color', palette={'White': '#d62728', 'Black': '#1f77b4'}, dodge=True, - linewidth=1.5, fliersize=4 + linewidth=1.5, fliersize=4, hue_order=['White', 'Black'] ) plt.xlabel('Głębokość przeszukiwania drzewa gry', fontsize=12) @@ -137,7 +199,7 @@ def generate_plots(df: pd.DataFrame, mcts_name: str): plt.savefig(moves_filename, dpi=300) plt.close() - print(f"Plots saved:") + print("Plots saved:") print(f" -> {mcts_filename}") print(f" -> {moves_filename}") @@ -152,11 +214,13 @@ def main(): print("Error: Input file(s) not found.") return - df, mcts_name, minimax_name = load_and_merge_data(args.white, args.black) + df_games, df_moves, mcts_name, minimax_name = load_all_data(args.white, args.black) + + print(f"Processed {len(df_games)} games.") - print(f"Processed {len(df)} games.") - print_statistics(df, mcts_name, minimax_name) - generate_plots(df, mcts_name) + print_statistics(df_games, mcts_name, minimax_name) + generate_plots(df_games, mcts_name) + generate_move_time_plots(df_moves, mcts_name, minimax_name) if __name__ == "__main__": main() \ No newline at end of file