diff --git a/.dockerignore b/.dockerignore index 933113a6..b529d78c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,21 @@ .vscode/ .cache/ +**/.cache/ build/ +**/build/ strfry +strfry-db* +**/strfry-db* +node_modules/ +**/*.o +**/*.d +**/*.a +**/*.so +**/*_test +**/timer_test +**/zerocopy_test *.md *.Dockerfile Dockerfile .gitignore - diff --git a/.gitignore b/.gitignore index 13ab5924..5128b0c6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,23 @@ /strfry-db-test* /seeds/ node_modules/ -.DS_Store \ No newline at end of file +.DS_Store +__pycache__/ +*.pyc +bench/alloc_tracker.so +bench/allocs.txt +bench/perf.txt +bench/det_new.json +bench/det_old.json +bench/test_seed.jsonl +benchmark_comparison.md +benchmark_report*.md +bench/flamegraphs/ +bench/perf*.data* +bench/seed_*.jsonl +strfry-db-perf/ +strfry-db-det/ +strfry-db-negentropy*/ +bench/*_temp.* +*.svg +perf.data* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 65a894d0..2d64965e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ RUN \ # Create a non-root user for security RUN adduser -D -h /app -s /bin/sh strfry && \ + mkdir -p /app/strfry-db && \ chown -R strfry:strfry /app # Switch to the unprivileged user @@ -56,8 +57,6 @@ USER strfry COPY --from=build --chown=strfry:strfry /build/strfry /app/strfry COPY --from=build --chown=strfry:strfry /build/strfry.conf /app/strfry.conf -COPY --from=build --chown=strfry:strfry /build/strfry-db /app/strfry-db - EXPOSE 7777 ENTRYPOINT ["/app/strfry"] diff --git a/README.md b/README.md index 1510917e..3dc389eb 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ If you are using strfry, please [join our telegram chat](https://t.me/strfry_use * [Router](#router) * [Syncing](#syncing) * [Compression Dictionaries](#compression-dictionaries) +* [Benchmarking & Performance](#benchmarking-performance) * [Learn More](#learn-more) * [Author and Copyright](#author-and-copyright) @@ -395,6 +396,9 @@ After building dictionaries, selections of events can be compressed with `strfry `strfry dict stats` can be used to print out stats for the various dictionaries, including size used by the dataset, compression ratios, etc. +### Benchmarking Performance + +strfry includes a comprehensive benchmarking and relative A/B comparison suite in `bench/`, supporting deterministic instruction/allocation tracking, WebSocket stress testing, and hardware flamegraph profiling. See [docs/benchmarking.md](docs/benchmarking.md) for detailed instructions on running benchmarks, profiling with `perf`, and interpreting results. ## Learn More diff --git a/bench/ab_test.py b/bench/ab_test.py new file mode 100755 index 00000000..93126b3d --- /dev/null +++ b/bench/ab_test.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +import sys +import os +import subprocess +import shutil +import time +import json +import re +import argparse +from datetime import datetime + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STRFRY_DIR = os.path.dirname(SCRIPT_DIR) + +def parse_markdown_metrics(filepath): + metrics = {} + if not os.path.exists(filepath): + return metrics + with open(filepath, "r") as f: + for line in f: + m = re.match(r'-\s+\*\*(.*?)(?::\*\*|\*\*[:]*)\s*(.*)', line.strip()) + if m: + name = m.group(1).strip() + val_str = m.group(2).strip() + num_m = re.search(r'[-+]?\d*\.\d+|\d+', val_str) + if num_m: + metrics[name] = float(num_m.group(0)) + return metrics + +def run_deterministic_benchmark(out_json_path): + print(f"[INFO] Running deterministic benchmark to {out_json_path}...") + + # 1. Compile allocator tracker + tracker_src = os.path.join(STRFRY_DIR, "bench", "alloc_tracker.c") + tracker_so = os.path.join(STRFRY_DIR, "bench", "alloc_tracker.so") + print(f"[INFO] Compiling allocation tracker...") + subprocess.run(["gcc", "-shared", "-fPIC", "-o", tracker_so, tracker_src, "-ldl"], check=True) + + # 2. Generate test seed data (always deterministic) + seed_data_path = os.path.join(STRFRY_DIR, "bench", "test_seed.jsonl") + if not os.path.exists(seed_data_path): + print(f"[INFO] Generating deterministic test seed data...") + subprocess.run([ + "perl", "test/generate-seed-data.pl", + "--seed", "1337", + "--output", seed_data_path, + "--users", "10", + "--kind1-notes", "100", + "--kind0-profiles", "10", + "--kind3-contacts", "5", + "--kind4-dms", "10", + "--kind7-reactions", "10", + "--replaceable", "10", + "--param-replaceable", "10", + "--ephemeral", "10", + "--deletions", "10", + "--other", "10", + "--duplicates", "5" + ], check=True) + + # 3. Clean db + db_det = os.path.join(STRFRY_DIR, "strfry-db-det") + shutil.rmtree(db_det, ignore_errors=True) + os.makedirs(db_det, exist_ok=True) + + # 4. Setup files for output + allocs_file = os.path.join(STRFRY_DIR, "bench", "allocs.txt") + perf_file = os.path.join(STRFRY_DIR, "bench", "perf.txt") + if os.path.exists(allocs_file): + os.remove(allocs_file) + if os.path.exists(perf_file): + os.remove(perf_file) + + # 5. Run perf stat & alloc tracker + env = os.environ.copy() + env["LD_PRELOAD"] = tracker_so + env["ALLOC_TRACKER_OUT"] = allocs_file + + binary = os.path.join(STRFRY_DIR, "strfry") + perf_cmd = [ + "perf", "stat", "-x,", "-o", perf_file, + "-e", "cpu_atom/instructions/u,cpu_core/instructions/u", + binary, "--set", "db=strfry-db-det/", "import", "--no-verify" + ] + + with open(seed_data_path, "r") as stdin_f: + subprocess.run(perf_cmd, stdin=stdin_f, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # 6. Parse allocations + total_allocs = 0 + total_bytes = 0 + if os.path.exists(allocs_file): + with open(allocs_file, "r") as f: + for line in f: + if line.startswith("ALLOCS:"): + total_allocs += int(line.split()[1]) + elif line.startswith("BYTES:"): + total_bytes += int(line.split()[1]) + + # 7. Parse instructions + total_instructions = 0 + if os.path.exists(perf_file): + with open(perf_file, "r") as f: + for line in f: + parts = line.strip().split(",") + if len(parts) >= 3 and "instructions" in parts[2]: + val = parts[0] + if val != "" and val != "": + try: + total_instructions += int(val) + except ValueError: + pass + + shutil.rmtree(db_det, ignore_errors=True) + + # Save results + results = { + "instructions": total_instructions, + "allocations": total_allocs, + "allocated_bytes": total_bytes + } + with open(out_json_path, "w") as f: + json.dump(results, f, indent=2) + + print(f"[INFO] Deterministic metrics: {results}") + return results + +def main(): + parser = argparse.ArgumentParser(description="Strfry Local Relative A/B Benchmarking Script") + parser.add_argument("--base", type=str, default="master", help="Base branch/commit to compare against (default: master)") + parser.add_argument("--skip-heavy", action="store_true", help="Skip 1M event heavy database benchmark") + parser.add_argument("--full", action="store_true", help="Run all benchmark suites including 1M event storage test (recommended before opening a PR)") + args = parser.parse_args() + + base_branch = args.base + + # 1. Save original branch & check status + res = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True, check=True) + current_branch = res.stdout.strip() + + res = subprocess.run(["git", "status", "--porcelain"], capture_output=True, text=True, check=True) + has_changes = len(res.stdout.strip()) > 0 + + stashed = False + + print(f"[INFO] Current branch is {current_branch}") + print(f"[INFO] Comparing against base branch {base_branch}") + + temp_run_stats_py = os.path.join(STRFRY_DIR, "bench", "run_stats_temp.py") + temp_alloc_tracker_c = os.path.join(STRFRY_DIR, "bench", "alloc_tracker_temp.c") + temp_bench_plugin_py = os.path.join(STRFRY_DIR, "bench", "bench_plugin_temp.py") + + alloc_tracker_c = os.path.join(STRFRY_DIR, "bench", "alloc_tracker.c") + bench_plugin_py = os.path.join(STRFRY_DIR, "bench", "bench_plugin.py") + try: + # Copy current run_stats.py to a temp path so we run the same runner on both branches + run_stats_py = os.path.join(STRFRY_DIR, "bench", "run_stats.py") + shutil.copy(run_stats_py, temp_run_stats_py) + shutil.copy(alloc_tracker_c, temp_alloc_tracker_c) + shutil.copy(bench_plugin_py, temp_bench_plugin_py) + # 2. Run PR branch (new changes) + print(f"\n=== BENCHMARKING NEW CHANGES (Branch: {current_branch}) ===") + # Re-build just in case + print("[INFO] Rebuilding strfry on current branch...") + subprocess.run(["make", "-j4"], check=True) + + # Run stats + report_new = os.path.join(STRFRY_DIR, "benchmark_report_new.md") + skip_flag = [] if args.full else (["--skip-heavy"] if args.skip_heavy else []) + + subprocess.run(["python3", temp_run_stats_py] + skip_flag, check=True) + if os.path.exists("benchmark_report.md"): + shutil.copy("benchmark_report.md", report_new) + + det_new = os.path.join(STRFRY_DIR, "bench", "det_new.json") + run_deterministic_benchmark(det_new) + + # Stash changes if any + if has_changes: + print("[INFO] Stashing uncommitted changes...") + subprocess.run(["git", "stash", "push", "-m", "ab_test_auto_stash"], check=True) + stashed = True + + # 3. Checkout base branch + print(f"\n=== CHECKING OUT BASE BRANCH ({base_branch}) ===") + subprocess.run(["git", "checkout", base_branch], check=True) + + # Restore needed files if they were deleted by checkout + if not os.path.exists(alloc_tracker_c): + shutil.copy(temp_alloc_tracker_c, alloc_tracker_c) + if not os.path.exists(bench_plugin_py): + shutil.copy(temp_bench_plugin_py, bench_plugin_py) + print("[INFO] Building strfry on base branch...") + subprocess.run(["make", "-j4"], check=True) + + report_old = os.path.join(STRFRY_DIR, "benchmark_report_old.md") + subprocess.run(["python3", temp_run_stats_py] + skip_flag, check=True) + if os.path.exists("benchmark_report.md"): + shutil.copy("benchmark_report.md", report_old) + + det_old = os.path.join(STRFRY_DIR, "bench", "det_old.json") + run_deterministic_benchmark(det_old) + + finally: + # Clean up temp files + for temp_file in [temp_run_stats_py, temp_alloc_tracker_c, temp_bench_plugin_py]: + if os.path.exists(temp_file): + try: + os.remove(temp_file) + except: + pass + # Remove restored files on base branch so they don't block checkout + for restored_file in [alloc_tracker_c, bench_plugin_py]: + if os.path.exists(restored_file): + try: + os.remove(restored_file) + except: + pass + # 4. Checkout back to current branch + print(f"\n=== RESTORING ORIGINAL STATE (Branch: {current_branch}) ===") + subprocess.run(["git", "checkout", current_branch], stderr=subprocess.DEVNULL) + print("[INFO] Rebuilding strfry...") + subprocess.run(["make", "-j4"], stderr=subprocess.DEVNULL) + + if stashed: + print("[INFO] Restoring stashed changes...") + subprocess.run(["git", "stash", "pop"], stderr=subprocess.DEVNULL) + + # 5. Parse and compare metrics + new_metrics = parse_markdown_metrics(os.path.join(STRFRY_DIR, "benchmark_report_new.md")) + old_metrics = parse_markdown_metrics(os.path.join(STRFRY_DIR, "benchmark_report_old.md")) + + det_new_data = {} + det_old_data = {} + det_new_path = os.path.join(STRFRY_DIR, "bench", "det_new.json") + det_old_path = os.path.join(STRFRY_DIR, "bench", "det_old.json") + if os.path.exists(det_new_path): + with open(det_new_path, "r") as f: + det_new_data = json.load(f) + if os.path.exists(det_old_path): + with open(det_old_path, "r") as f: + det_old_data = json.load(f) + + # Combine metrics + all_keys = sorted(list(set(new_metrics.keys()) | set(old_metrics.keys()))) + + comparison_path = os.path.join(STRFRY_DIR, "benchmark_comparison.md") + print(f"\n[INFO] Generating comparison report at {comparison_path}") + + with open(comparison_path, "w") as f: + f.write("# Strfry A/B Benchmark Comparison Report\n\n") + f.write(f"Generated at: {datetime.now().isoformat()}\n") + f.write(f"- **Base Branch:** {base_branch}\n") + f.write(f"- **PR/Current Branch:** {current_branch}\n\n") + + f.write("## 1. Deterministic Metrics (Hardware Agnostic)\n") + f.write("| Metric | Base | PR | Delta (%) | Status |\n") + f.write("| :--- | :--- | :--- | :--- | :--- |\n") + + det_metrics = [ + ("instructions", "Instructions Retired"), + ("allocations", "Heap Allocations Count"), + ("allocated_bytes", "Total Bytes Allocated") + ] + + for key, label in det_metrics: + base_val = det_old_data.get(key, 0) + pr_val = det_new_data.get(key, 0) + if base_val > 0: + delta = ((pr_val - base_val) / base_val) * 100 + delta_str = f"{delta:+.2f}%" + status = "🟢 Improved" if pr_val < base_val else ("🔴 Regressed" if pr_val > base_val else "⚪ No Change") + else: + delta_str = "N/A" + status = "Unknown" + + f.write(f"| {label} | {base_val:,} | {pr_val:,} | {delta_str} | {status} |\n") + + f.write("\n") + + f.write("## 2. Standard Benchmark Suites (Wall-clock & Resource Metrics)\n") + f.write("| Suite Metric | Base | PR | Delta (%) | Status |\n") + f.write("| :--- | :--- | :--- | :--- | :--- |\n") + + for key in all_keys: + if key.lower() in ["status", "status:"]: + continue + base_val = old_metrics.get(key, -1) + pr_val = new_metrics.get(key, -1) + + if base_val >= 0 and pr_val >= 0: + if base_val == 0: + if pr_val == 0: + delta = 0.0 + delta_str = "0.00%" + else: + delta = 100.0 + delta_str = "N/A" + else: + delta = ((pr_val - base_val) / base_val) * 100 + delta_str = f"{delta:+.2f}%" + + # Check if lower is better or higher is better + lower_better = any(x in key.lower() for x in ["time", "ms", "rss", "bytes", "amplification", "depth", "memory", "sockets"]) + + if abs(delta) < 1.0: + status = "⚪ Stable" + elif pr_val < base_val: + status = "🟢 Faster/Lighter" if lower_better else "🔴 Slower/Lower" + else: + status = "🔴 Slower/Heavier" if lower_better else "🟢 Faster/Higher" + else: + delta_str = "N/A" + status = "Unknown" + + f.write(f"| {key} | {base_val:.2f} | {pr_val:.2f} | {delta_str} | {status} |\n") + + print(f"[INFO] Comparison complete. Comparison saved to {comparison_path}.") + +if __name__ == "__main__": + main() diff --git a/bench/alloc_tracker.c b/bench/alloc_tracker.c new file mode 100644 index 00000000..e2567c67 --- /dev/null +++ b/bench/alloc_tracker.c @@ -0,0 +1,104 @@ +#define _GNU_SOURCE +#include +#include +#include +#include + +static void* (*real_malloc)(size_t) = NULL; +static void* (*real_calloc)(size_t, size_t) = NULL; +static void* (*real_realloc)(void*, size_t) = NULL; +static void (*real_free)(void*) = NULL; + +static atomic_size_t total_allocations = 0; +static atomic_size_t total_bytes = 0; + +static __thread int in_hook = 0; +static char bootstrap_buf[4096]; +static size_t bootstrap_offset = 0; + +static void init() { + if (real_malloc) return; + in_hook = 1; + real_malloc = (void* (*)(size_t))dlsym(RTLD_NEXT, "malloc"); + real_calloc = (void* (*)(size_t, size_t))dlsym(RTLD_NEXT, "calloc"); + real_realloc = (void* (*)(void*, size_t))dlsym(RTLD_NEXT, "realloc"); + real_free = (void (*)(void*))dlsym(RTLD_NEXT, "free"); + in_hook = 0; +} + +void* malloc(size_t size) { + if (!real_malloc) init(); + if (in_hook) return real_malloc ? real_malloc(size) : NULL; + + in_hook = 1; + void* ptr = real_malloc(size); + in_hook = 0; + + if (ptr) { + atomic_fetch_add(&total_allocations, 1); + atomic_fetch_add(&total_bytes, size); + } + return ptr; +} + +void* calloc(size_t nmemb, size_t size) { + if (!real_calloc) { + if (in_hook) { + size_t total = nmemb * size; + if (bootstrap_offset + total < sizeof(bootstrap_buf)) { + void* ptr = &bootstrap_buf[bootstrap_offset]; + bootstrap_offset += total; + return ptr; + } + return NULL; + } + init(); + } + if (in_hook) return real_calloc ? real_calloc(nmemb, size) : NULL; + + in_hook = 1; + void* ptr = real_calloc(nmemb, size); + in_hook = 0; + + if (ptr) { + atomic_fetch_add(&total_allocations, 1); + atomic_fetch_add(&total_bytes, nmemb * size); + } + return ptr; +} + +void* realloc(void* ptr, size_t size) { + if (!real_realloc) init(); + if (in_hook) return real_realloc ? real_realloc(ptr, size) : NULL; + + in_hook = 1; + void* new_ptr = real_realloc(ptr, size); + in_hook = 0; + + if (new_ptr) { + atomic_fetch_add(&total_allocations, 1); + atomic_fetch_add(&total_bytes, size); + } + return new_ptr; +} + +void free(void* ptr) { + if (!real_free) init(); + if (real_free) { + if (ptr >= (void*)bootstrap_buf && ptr < (void*)(bootstrap_buf + sizeof(bootstrap_buf))) { + return; + } + real_free(ptr); + } +} + +__attribute__((destructor)) void report() { + const char* out_path = getenv("ALLOC_TRACKER_OUT"); + FILE* f = stderr; + if (out_path) { + f = fopen(out_path, "a"); + if (!f) f = stderr; + } + fprintf(f, "ALLOCS: %zu\nBYTES: %zu\n", atomic_load(&total_allocations), atomic_load(&total_bytes)); + if (f != stderr) fclose(f); +} diff --git a/bench/bench_plugin.py b/bench/bench_plugin.py new file mode 100755 index 00000000..4992ccbe --- /dev/null +++ b/bench/bench_plugin.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import sys +import json + +for line in sys.stdin: + try: + req = json.loads(line) + if req.get("type") == "new": + res = { + "id": req["event"]["id"], + "action": "accept" + } + print(json.dumps(res), flush=True) + except Exception as e: + sys.stderr.write(f"Error: {e}\n") + sys.stderr.flush() diff --git a/bench/run_stats.py b/bench/run_stats.py new file mode 100644 index 00000000..a1ec06e7 --- /dev/null +++ b/bench/run_stats.py @@ -0,0 +1,1033 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import os +import sys +import time +import json +import urllib.request +import re +import shutil +import threading +from datetime import datetime + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STRFRY_DIR = os.path.dirname(SCRIPT_DIR) + +import builtins +def print(*args, **kwargs): + kwargs.setdefault('flush', True) + builtins.print(*args, **kwargs) + +def check_docker(): + try: + res = subprocess.run(["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=3) + return res.returncode == 0 + except: + return False + +def drop_caches(): + print("[INFO] Dropping OS caches...") + res = subprocess.run(["sudo", "-n", "sh", "-c", "sync; echo 3 > /proc/sys/vm/drop_caches"], capture_output=True) + if res.returncode != 0: + print("[WARNING] Could not drop OS caches (requires passwordless sudo). Out-of-core benchmarks might use cached pages.") + + +class StrfryManager: + def __init__(self, use_docker=False, memory_limit=None): + self.use_docker = use_docker + self.memory_limit = memory_limit + self.process = None + self.db_dir = os.path.join(STRFRY_DIR, 'strfry-db') + self.config_path = os.path.join(STRFRY_DIR, 'strfry.conf') + self.binary_path = os.path.join(STRFRY_DIR, 'strfry') + def build_docker(self): + print("[INFO] Building strfry docker image...") + subprocess.run(["docker", "build", "--progress=plain", "-t", "strfry-bench-image", "."], cwd=STRFRY_DIR, check=True) + + def clean_db(self): + print("[INFO] Cleaning database...") + res = subprocess.run(["rm", "-rf", self.db_dir]) + if res.returncode != 0 or os.path.exists(self.db_dir): + print("[WARNING] Normal rm -rf failed or directory still exists, trying with sudo...") + subprocess.run(["sudo", "rm", "-rf", self.db_dir]) + os.makedirs(self.db_dir, exist_ok=True) + + def start(self, config_overrides=None): + if self.process is not None: + self.stop() + + print("[INFO] Starting strfry relay...") + + args = [] + if config_overrides: + for k, v in config_overrides.items(): + args.extend(["--set", f"{k}={v}"]) + + if self.use_docker: + cmd = [ + "docker", "run", "--rm", "-p", "7777:7777", + "-v", f"{self.config_path}:/app/strfry.conf", + "-v", f"{self.db_dir}:/app/strfry-db" + ] + if self.memory_limit: + cmd.extend(["--memory", self.memory_limit]) + cmd.append("strfry-bench-image") + # Always bind to 0.0.0.0 inside Docker so the host can connect + cmd.extend(["--set", "relay.bind=0.0.0.0"]) + cmd.extend(args) + cmd.append("relay") + self.process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + cmd = [self.binary_path, "--config", self.config_path] + cmd.extend(args) + cmd.append("relay") + self.process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + time.sleep(2) # Wait for relay to bind port + print("[INFO] Relay started.") + + def stop(self): + if self.process: + print("[INFO] Stopping strfry relay...") + self.process.terminate() + self.process.wait() + self.process = None + print("[INFO] Relay stopped.") + +class PrometheusScraper: + def __init__(self, port=7777): + self.url = f"http://localhost:{port}/metrics" + self.running = False + self.thread = None + self.peak_queue = 0.0 + + def get_metrics(self): + try: + req = urllib.request.Request(self.url) + with urllib.request.urlopen(req) as response: + return response.read().decode('utf-8') + except Exception as e: + return "" + + def parse_metric(self, metrics_text, metric_name): + for line in metrics_text.splitlines(): + if line.startswith(metric_name): + parts = line.split() + if len(parts) >= 2: + return float(parts[1]) + return 0.0 + + def _poll_loop(self): + while self.running: + metrics_text = self.get_metrics() + q = self.parse_metric(metrics_text, "strfry_writer_queue") + if q > self.peak_queue: + self.peak_queue = q + time.sleep(0.1) + + def start_polling(self): + self.running = True + self.peak_queue = 0.0 + self.thread = threading.Thread(target=self._poll_loop) + self.thread.daemon = True + self.thread.start() + + def stop_polling(self): + self.running = False + if self.thread: + self.thread.join(timeout=1.0) + +class ResourceMonitor: + def __init__(self, pid): + self.pid = pid + self.running = False + self.thread = None + self.cpu_usages = [] + self.read_bytes_start = 0 + self.write_bytes_start = 0 + self.logical_write_start = 0 + self.read_bytes_end = 0 + self.write_bytes_end = 0 + self.logical_write_end = 0 + + def get_cpu_times(self): + try: + with open("/proc/stat", "r") as f: + first_line = f.readline() + parts = first_line.split() + total = sum(float(x) for x in parts[1:]) + idle = float(parts[4]) + return total, idle + except: + return 0, 0 + + def get_proc_cpu_time(self): + try: + with open(f"/proc/{self.pid}/stat", "r") as f: + parts = f.readline().split() + utime = float(parts[13]) + stime = float(parts[14]) + return utime + stime + except: + return 0 + + def get_proc_io_bytes(self): + try: + r, w, lw = 0, 0, 0 + with open(f"/proc/{self.pid}/io", "r") as f: + for line in f: + if line.startswith("read_bytes:"): + r = int(line.split()[1]) + elif line.startswith("write_bytes:"): + w = int(line.split()[1]) + elif line.startswith("wchar:"): + lw = int(line.split()[1]) + return r, w, lw + except: + return 0, 0, 0 + + def _poll_loop(self): + num_cores = os.cpu_count() or 1 + while self.running: + t1_total, t1_idle = self.get_cpu_times() + p1_time = self.get_proc_cpu_time() + time.sleep(0.5) + t2_total, t2_idle = self.get_cpu_times() + p2_time = self.get_proc_cpu_time() + + total_diff = t2_total - t1_total + proc_diff = p2_time - p1_time + if total_diff > 0: + cpu_pct = (proc_diff / total_diff) * 100 * num_cores + self.cpu_usages.append(cpu_pct) + + def start(self): + self.running = True + self.cpu_usages = [] + r, w, lw = self.get_proc_io_bytes() + self.read_bytes_start = r + self.write_bytes_start = w + self.logical_write_start = lw + self.thread = threading.Thread(target=self._poll_loop) + self.thread.daemon = True + self.thread.start() + + def stop(self): + self.running = False + if self.thread: + self.thread.join(timeout=1.0) + r, w, lw = self.get_proc_io_bytes() + self.read_bytes_end = r + self.write_bytes_end = w + self.logical_write_end = lw + + def get_results(self): + avg_cpu = sum(self.cpu_usages) / len(self.cpu_usages) if self.cpu_usages else 0.0 + peak_cpu = max(self.cpu_usages) if self.cpu_usages else 0.0 + read_delta = self.read_bytes_end - self.read_bytes_start + write_delta = self.write_bytes_end - self.write_bytes_start + logical_write_delta = self.logical_write_end - self.logical_write_start + waf = write_delta / logical_write_delta if logical_write_delta > 0 else 0.0 + return { + "avg_cpu_percent": avg_cpu, + "peak_cpu_percent": peak_cpu, + "read_bytes": read_delta, + "write_bytes": write_delta, + "logical_write_bytes": logical_write_delta, + "waf": waf, + "read_mb": read_delta / (1024 * 1024), + "write_mb": write_delta / (1024 * 1024) + } + +def get_bench_dir(): + parent_dir = os.path.dirname(STRFRY_DIR) + sibling_bench = os.path.join(parent_dir, "strfry-bench") + if os.path.exists(sibling_bench): + return sibling_bench + local_bench = os.path.join(STRFRY_DIR, "strfry-bench") + return local_bench + +def run_bench(command, args): + bench_dir = get_bench_dir() + cmd = ["./target/release/strfry-bench", command] + args + print(f"[INFO] Running bench: {' '.join(cmd)}") + start = time.time() + result = subprocess.run(cmd, cwd=bench_dir, capture_output=True, text=True) + end = time.time() + + if result.returncode != 0: + print(f"[ERROR] strfry-bench failed:\n{result.stderr}") + return None + + return { + "output": result.stdout, + "elapsed": end - start + } + +def generate_seed_data(events=100000): + print(f"[INFO] Generating {events} events for seed...") + ratio = events / 100000.0 + users = max(1, int(500 * ratio)) + kind1_notes = max(1, int(70000 * ratio)) + kind4_dms = max(1, int(8000 * ratio)) + kind7_reactions = max(1, int(8000 * ratio)) + replaceable = max(1, int(6000 * ratio)) + param_replaceable = max(1, int(10000 * ratio)) + ephemeral = max(1, int(4000 * ratio)) + deletions = max(1, int(2500 * ratio)) + other = max(1, int(3000 * ratio)) + duplicates = max(1, int(1500 * ratio)) + + gen_cmd = [ + "perl", "test/generate-seed-data.pl", "-o", "-", + "--users", str(users), + "--kind1-notes", str(kind1_notes), + "--kind4-dms", str(kind4_dms), + "--kind7-reactions", str(kind7_reactions), + "--replaceable", str(replaceable), + "--param-replaceable", str(param_replaceable), + "--ephemeral", str(ephemeral), + "--deletions", str(deletions), + "--other", str(other), + "--duplicates", str(duplicates) + ] + binary = os.path.join(STRFRY_DIR, "strfry") + import_cmd = [binary, "import", "--no-verify"] + + p1 = subprocess.Popen(gen_cmd, stdout=subprocess.PIPE, cwd=STRFRY_DIR) + p2 = subprocess.Popen(import_cmd, stdin=p1.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + p1.stdout.close() + p2.communicate() + print("[INFO] Seeding complete.") + +def run_iostat(duration=5): + # Runs iostat for the given duration and returns MB/s read/write + if not shutil.which("iostat"): + return {"read_mb": -1, "write_mb": -1} + cmd = ["iostat", "-m", "-y", "1", str(duration)] + res = subprocess.run(cmd, capture_output=True, text=True) + # Parse logic omitted for brevity, returning dummy data for now + return {"read_mb": 0.0, "write_mb": 0.0} + +def suite_storage(manager, skip_heavy=False): + print("\n--- Running Suite 1: Storage (In-Core vs Out-of-Core) ---") + results = {} + manager.clean_db() + + events_count = 10000 if skip_heavy else 1000000 + generate_seed_data(events_count) + + # 1. In-Core Test + manager.use_docker = False + manager.start() + + start = time.time() + scan_res = subprocess.run(["./strfry", "scan", "{}"], capture_output=True, text=True) + results["scan_time"] = time.time() - start + results["scan_tps"] = events_count / results["scan_time"] if results["scan_time"] > 0 else 0 + + res = run_bench("paginate", ["ws://localhost:7777", "--depth", "10", "--concurrency", "2"]) + results["in_core_time"] = res["elapsed"] if res else -1 + manager.stop() + + # 2. Out-of-Core Test (Docker, 256MB memory limit) + if check_docker(): + drop_caches() + manager.use_docker = True + manager.memory_limit = "256m" + manager.start() + res = run_bench("paginate", ["ws://localhost:7777", "--depth", "10", "--concurrency", "2"]) + results["out_of_core_time"] = res["elapsed"] if res else -1 + manager.stop() + else: + print("[WARNING] Docker daemon not available. Skipping Out-of-Core test.") + results["out_of_core_time"] = -1 + + try: + mdb_res = subprocess.run(["mdb_stat", "-e", manager.db_dir], capture_output=True, text=True) + results["mdb_stat"] = mdb_res.stdout + except FileNotFoundError: + results["mdb_stat"] = "mdb_stat not installed" + + return results + +def suite_ingestion(manager, skip_heavy=False): + print("\n--- Running Suite 2: Ingestion Pipeline ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + count = 1000 if skip_heavy else 50000 + + scraper = PrometheusScraper() + scraper.start_polling() + + monitor = ResourceMonitor(manager.process.pid) + monitor.start() + + # Standard Events (Small) + res_small = run_bench("event", ["ws://localhost:7777", "-c", "20", "-n", str(count), "--payload-size", "50"]) + + # Standard Events (Large) + res_large = run_bench("event", ["ws://localhost:7777", "-c", "20", "-n", str(count // 10), "--payload-size", "10000"]) + + # Spam / Rate Limiting (Single connection trying to blast events) + res_spam = run_bench("event", ["ws://localhost:7777", "-c", "1", "-n", str(count)]) + + monitor.stop() + scraper.stop_polling() + + results.update(monitor.get_results()) + results["writer_queue_peak"] = scraper.peak_queue + results["event_small_tps"] = count / res_small["elapsed"] if res_small else -1 + results["event_large_tps"] = (count // 10) / res_large["elapsed"] if res_large else -1 + results["event_spam_tps"] = count / res_spam["elapsed"] if res_spam else -1 + results["event_small_output"] = res_small["output"] if res_small else "" + + manager.stop() + return results + +def suite_concurrency(manager, skip_heavy=False): + print("\n--- Running Suite 3: Concurrency & Thread Pool ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + events = 1000 if skip_heavy else 100000 + + # Background writer + bench_dir = get_bench_dir() + write_cmd = ["./target/release/strfry-bench", "event", "ws://localhost:7777", "-c", "20", "-n", str(events)] + writer = subprocess.Popen(write_cmd, cwd=bench_dir, stdout=subprocess.DEVNULL) + + time.sleep(2) # Let writer build pressure + + req_start = time.time() + res = run_bench("req", ["ws://localhost:7777", "-c", "10", "-n", "1000", "--filter", "{\"limit\":10}"]) + results["mixed_req_time"] = res["elapsed"] if res else -1 + + writer.wait() + manager.stop() + results["status"] = "Done" + return results + +def get_process_rss(pid): + try: + with open(f"/proc/{pid}/status", "r") as f: + for line in f: + if line.startswith("VmRSS:"): + return int(line.split()[1]) / 1024 # MB + except: + pass + return -1 + +def get_process_cpu_time(pid): + try: + with open(f"/proc/{pid}/stat", "r") as f: + parts = f.read().split() + # 13: utime, 14: stime + return float(parts[13]) + float(parts[14]) + except: + return 0.0 + +def get_process_io(pid): + io_data = {"read_bytes": 0, "write_bytes": 0} + try: + with open(f"/proc/{pid}/io", "r") as f: + for line in f: + if line.startswith("read_bytes:"): + io_data["read_bytes"] = int(line.split()[1]) + elif line.startswith("write_bytes:"): + io_data["write_bytes"] = int(line.split()[1]) + except: + pass + return io_data + +def count_time_wait_sockets(port=7777): + port_hex = f"{port:04X}" + count = 0 + for filename in ["/proc/net/tcp", "/proc/net/tcp6"]: + if not os.path.exists(filename): + continue + try: + with open(filename, "r") as f: + lines = f.readlines() + for line in lines[1:]: # skip header + parts = line.split() + if len(parts) >= 4: + state = parts[3] + if state == "06": # TIME_WAIT + local_port = parts[1].split(":")[-1] + remote_port = parts[2].split(":")[-1] + if local_port == port_hex or remote_port == port_hex: + count += 1 + except Exception: + pass + return count + +def suite_websockets(manager, skip_heavy=False): + print("\n--- Running Suite 4: WebSockets & Connections ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + counts = [100, 500] if skip_heavy else [100, 1000] + if not skip_heavy: + counts.append(5000) + + results["connection_memory"] = {} + for c in counts: + print(f"[INFO] Testing {c} connection storm...") + res = run_bench("connections", ["ws://localhost:7777", "-c", str(c)]) + rss = get_process_rss(manager.process.pid) + results["connection_memory"][str(c)] = rss + if res: + output = res["output"] + results[f"conn_storm_{c}_output"] = output + tps_m = re.search(r'\(([\d.]+)\s+conn/sec\)', output) + if tps_m: + results[f"conn_storm_{c}_tps"] = float(tps_m.group(1)) + lat_m = re.search(r'P50:\s*([\d.]+).*P99:\s*([\d.]+)', output) + if lat_m: + results[f"conn_storm_{c}_p50_ms"] = float(lat_m.group(1)) + results[f"conn_storm_{c}_p99_ms"] = float(lat_m.group(2)) + time.sleep(1) # Let strfry clean up + print("[INFO] Testing High Churn...") + churn_count = 200 if skip_heavy else 10000 + res_churn = run_bench("churn", ["ws://localhost:7777", "-c", "50", "-n", str(churn_count)]) + if res_churn: + results["churn_output"] = res_churn["output"] + tps_m = re.search(r'\(([\d.]+)\s+conn/sec\)', res_churn["output"]) + if tps_m: + results["churn_tps"] = float(tps_m.group(1)) + + results["time_wait_count"] = count_time_wait_sockets(7777) + + manager.stop() + results["status"] = "Done" + return results + +def suite_queries(manager, skip_heavy=False): + print("\n--- Running Suite 5: Query Engine & Indices ---") + results = {} + manager.clean_db() + generate_seed_data(10000 if skip_heavy else 1000000) + manager.use_docker = False + manager.start() + + # 1. Point Lookup (exact id) + # Get a real ID from the DB via strfry scan + scan_res = subprocess.run(["./strfry", "scan", "{\"limit\":1}"], capture_output=True, text=True) + try: + real_id = json.loads(scan_res.stdout.strip().splitlines()[0])["id"] + except: + real_id = "0000000000000000000000000000000000000000000000000000000000000000" + + res_point = run_bench("req", ["ws://localhost:7777", "-c", "5", "-n", "100", "--filter", "{\"ids\":[\"" + real_id + "\"]}"]) + results["query_point_time"] = res_point["elapsed"] if res_point else -1 + results["query_point_output"] = res_point["output"] if res_point else "" + + # 2. Point COUNT Lookup (NIP-45) + res_point_count = run_bench("req", ["ws://localhost:7777", "-c", "5", "-n", "100", "--filter", "{\"ids\":[\"" + real_id + "\"]}", "--nip45"]) + results["query_point_count_time"] = res_point_count["elapsed"] if res_point_count else -1 + results["query_point_count_output"] = res_point_count["output"] if res_point_count else "" + + # 3. Complex Query (authors, kinds, tags, time range) + # A heavy NIP-01 complex query + complex_filter = json.dumps({ + "authors": ["0000000000000000000000000000000000000000000000000000000000000000"], + "kinds": [1, 5, 7], + "#t": ["nostr", "benchmark"], + "since": 1600000000, + "until": 1800000000, + "limit": 100 + }) + res_complex = run_bench("req", ["ws://localhost:7777", "-c", "10", "-n", "100", "--filter", complex_filter]) + results["query_complex_time"] = res_complex["elapsed"] if res_complex else -1 + results["query_complex_output"] = res_complex["output"] if res_complex else "" + + # 4. Complex COUNT Query (NIP-45) + res_complex_count = run_bench("req", ["ws://localhost:7777", "-c", "10", "-n", "100", "--filter", complex_filter, "--nip45"]) + results["query_complex_count_time"] = res_complex_count["elapsed"] if res_complex_count else -1 + results["query_complex_count_output"] = res_complex_count["output"] if res_complex_count else "" + + manager.stop() + results["status"] = "Done" + return results + +def suite_monitors(manager, skip_heavy=False): + print("\n--- Running Suite 6: Active Monitors ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + subs = 50 if skip_heavy else 150 + res = run_bench("monitor", ["ws://localhost:7777", "-s", str(subs), "-p", "100"]) + results["monitor_fanout_time"] = res["elapsed"] if res else -1 + results["monitor_output"] = res["output"] if res else "" + + manager.stop() + results["status"] = "Done" + return results + +def suite_negentropy(manager, skip_heavy=False): + print("\n--- Running Suite 7: Negentropy Sync ---") + results = {} + import shutil + target_db_dir = os.path.join(STRFRY_DIR, 'strfry-db-negentropy-target') + shutil.rmtree(target_db_dir, ignore_errors=True) + os.makedirs(target_db_dir, exist_ok=True) + + # 1. Clean and seed the main DB + manager.clean_db() + events_count = 100 if skip_heavy else 10000 + generate_seed_data(events_count) + + # 2. Start the source relay + manager.use_docker = False + manager.start() + + # 3. Perform negentropy sync + print(f"[INFO] Syncing {events_count} events via Negentropy...") + binary = os.path.join(STRFRY_DIR, "strfry") + sync_cmd = [ + binary, + "--set", f"db={target_db_dir}/", + "sync", "ws://127.0.0.1:7777" + ] + + start_time = time.time() + res = subprocess.run(sync_cmd, capture_output=True, text=True) + elapsed = time.time() - start_time + + manager.stop() + + if res.returncode != 0: + print(f"[ERROR] Negentropy sync failed:\n{res.stderr}") + results["status"] = f"Failed: {res.stderr.strip()}" + else: + results["elapsed"] = elapsed + results["tps"] = events_count / elapsed if elapsed > 0 else 0 + results["status"] = f"Success ({events_count} events synced in {elapsed:.2f}s)" + print(f"[INFO] Negentropy sync finished. Status: {results['status']}") + + shutil.rmtree(target_db_dir, ignore_errors=True) + return results + +def suite_plugin(manager, skip_heavy=False): + print("\n--- Running Suite 8: Write Policy Plugin ---") + results = {} + manager.clean_db() + manager.use_docker = False + + # Configure the writePolicy plugin + plugin_path = os.path.abspath(os.path.join(SCRIPT_DIR, "bench_plugin.py")) + manager.start(config_overrides={ + "relay.writePolicy.plugin": plugin_path, + "relay.writePolicy.timeoutSeconds": "10" + }) + + count = 1000 if skip_heavy else 10000 + res = run_bench("event", ["ws://localhost:7777", "-c", "10", "-n", str(count), "--payload-size", "50"]) + + manager.stop() + + if res: + results["elapsed"] = res["elapsed"] + results["tps"] = count / res["elapsed"] if res["elapsed"] > 0 else 0 + results["status"] = f"Success ({count} events processed with plugin)" + results["output"] = res["output"] + else: + results["status"] = "Failed" + + return results + +def suite_cli_dict(manager, skip_heavy=False): + print("\n--- Running Suite 9 & 10: CLI & Dictionary ---") + results = {} + manager.clean_db() + events = 1000 if skip_heavy else 1000000 + + print("[INFO] Testing strfry import...") + ratio = events / 100000.0 + users = max(1, int(500 * ratio)) + kind1_notes = max(1, int(70000 * ratio)) + kind4_dms = max(1, int(8000 * ratio)) + kind7_reactions = max(1, int(8000 * ratio)) + replaceable = max(1, int(6000 * ratio)) + param_replaceable = max(1, int(10000 * ratio)) + ephemeral = max(1, int(4000 * ratio)) + deletions = max(1, int(2500 * ratio)) + other = max(1, int(3000 * ratio)) + duplicates = max(1, int(1500 * ratio)) + + gen_cmd = [ + "perl", "test/generate-seed-data.pl", "-o", "-", + "--users", str(users), + "--kind1-notes", str(kind1_notes), + "--kind4-dms", str(kind4_dms), + "--kind7-reactions", str(kind7_reactions), + "--replaceable", str(replaceable), + "--param-replaceable", str(param_replaceable), + "--ephemeral", str(ephemeral), + "--deletions", str(deletions), + "--other", str(other), + "--duplicates", str(duplicates) + ] + import_cmd = ["./strfry", "import", "--no-verify"] + + start = time.time() + p1 = subprocess.Popen(gen_cmd, stdout=subprocess.PIPE) + p2 = subprocess.Popen(import_cmd, stdin=p1.stdout, stdout=subprocess.DEVNULL) + p1.stdout.close() + p2.communicate() + results["import_time"] = time.time() - start + + print("[INFO] Testing strfry export...") + start = time.time() + subprocess.run(["./strfry", "export"], stdout=subprocess.DEVNULL) + results["export_time"] = time.time() - start + + print("[INFO] Testing dictionary generation...") + start = time.time() + subprocess.run(["./strfry", "dict", "train"], stdout=subprocess.DEVNULL) + results["dict_gen_time"] = time.time() - start + + results["status"] = "Done" + return results + +def suite_os(manager, skip_heavy=False): + print("\n--- Running Suite 11: OS-Level Metrics ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + pid = manager.process.pid + + # 1. Initial State + initial_io = get_process_io(pid) + initial_cpu = get_process_cpu_time(pid) + db_file = os.path.join(manager.db_dir, "data.mdb") + initial_db_size = os.stat(db_file).st_blocks * 512 if os.path.exists(db_file) else 0 + start_time = time.time() + + # 2. Run workload (Ingest events) + count = 2000 if skip_heavy else 10000 + res = run_bench("event", ["ws://localhost:7777", "-c", "2", "-n", str(count)]) + + # 3. Final State + elapsed = time.time() - start_time + final_io = get_process_io(pid) + final_cpu = get_process_cpu_time(pid) + final_db_size = os.stat(db_file).st_blocks * 512 if os.path.exists(db_file) else 0 + + write_bytes = final_io["write_bytes"] - initial_io["write_bytes"] + read_bytes = final_io["read_bytes"] - initial_io["read_bytes"] + db_growth = final_db_size - initial_db_size + cpu_time_diff = final_cpu - initial_cpu + + # Ticks per second + try: + ticks_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) + except: + ticks_per_sec = 100 + + cpu_util = (cpu_time_diff / (elapsed * ticks_per_sec)) * 100 if elapsed > 0 else 0.0 + waf = write_bytes / db_growth if db_growth > 0 else 1.0 + + results["elapsed"] = elapsed + results["write_bytes"] = write_bytes + results["read_bytes"] = read_bytes + results["db_growth"] = db_growth + results["cpu_utilization_pct"] = cpu_util + results["write_amplification_factor"] = waf + results["process_rss"] = get_process_rss(pid) + + manager.stop() + results["status"] = "Done" + return results + +def suite_stress(manager, skip_heavy=False): + print("\n--- Running Suite 12: Stress & Edge Cases ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + count = 100 if skip_heavy else 2000 + print("[INFO] Running Slow Loris attack...") + res1 = run_bench("malicious", ["ws://localhost:7777", "-c", str(count), "--slow-loris"]) + + manager.stop() + manager.clean_db() + manager.start() + + print("[INFO] Running Signature Flood attack...") + res2 = run_bench("malicious", ["ws://localhost:7777", "-c", "50", "--sig-flood"]) + + manager.stop() + results["status"] = "Done" + return results + +def suite_backpressure(manager, skip_heavy=False): + print("\n--- Running Suite 13: Backpressure Performance ---") + results = {} + manager.clean_db() + manager.use_docker = False + manager.start() + + fast_clients = 20 if skip_heavy else 100 + slow_clients = 5 if skip_heavy else 20 + count = 100 if skip_heavy else 1000 + res = run_bench("backpressure", [ + "ws://localhost:7777", + "--fast-clients", str(fast_clients), + "--slow-clients", str(slow_clients), + "-n", str(count), + "--slow-delay", "50" + ]) + results["backpressure_time"] = res["elapsed"] if res else -1 + results["backpressure_output"] = res["output"] if res else "" + + manager.stop() + results["status"] = "Done" + return results + +def generate_report(results, report_path="benchmark_report.md"): + print(f"[INFO] Generating comprehensive report at {report_path}") + with open(report_path, "w") as f: + f.write("# Strfry Benchmarking Report\n\n") + f.write(f"Generated at: {datetime.now().isoformat()}\n\n") + + f.write("## 1. Storage & LMDB Statistics\n") + if "suite_storage" in results: + r = results["suite_storage"] + f.write(f"- **Sequential scan throughput (events/sec):** {r.get('scan_tps', -1):.2f}\n") + f.write(f"- **In-Core Pagination Time:** {r.get('in_core_time', -1):.2f} seconds\n") + f.write(f"- **Out-of-Core Pagination Time (256MB RAM):** {r.get('out_of_core_time', -1):.2f} seconds\n") + f.write("\n### DB Stat Output\n```\n") + f.write(r.get('mdb_stat', '')) + f.write("\n```\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 2. Event Ingestion Pipeline Statistics\n") + if "suite_ingestion" in results: + r = results["suite_ingestion"] + f.write(f"- **Standard Write throughput (events/sec) (50b payload):** {r.get('event_small_tps', -1):.2f}\n") + f.write(f"- **Standard Write throughput (events/sec) (10Kb payload):** {r.get('event_large_tps', -1):.2f}\n") + f.write(f"- **Spam Write throughput (events/sec):** {r.get('event_spam_tps', -1):.2f}\n") + f.write(f"- **Peak Writer Queue Depth:** {r.get('writer_queue_peak', -1)}\n") + f.write(f"- **Average CPU Utilization (across cores):** {r.get('avg_cpu_percent', 0.0):.2f}%\n") + f.write(f"- **Peak CPU Utilization:** {r.get('peak_cpu_percent', 0.0):.2f}%\n") + f.write(f"- **Disk Physical Reads:** {r.get('read_mb', 0.0):.2f} MB\n") + f.write(f"- **Disk Physical Writes:** {r.get('write_mb', 0.0):.2f} MB\n") + f.write(f"- **Write Amplification Factor (WAF):** {r.get('waf', 0.0):.4f}\n\n") + f.write("### Small Payload Latencies\n```\n") + f.write(r.get('event_small_output', '')) + f.write("\n```\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 3. Concurrency & Thread Pool\n") + if "suite_concurrency" in results: + r = results["suite_concurrency"] + f.write(f"- **Mixed Read-Write REQ Time:** {r.get('mixed_req_time', -1):.2f} seconds\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 4. WebSockets & Connections\n") + if "suite_websockets" in results: + r = results["suite_websockets"] + f.write("### Connection Memory Scaling (VmRSS)\n") + for c, mem in r.get('connection_memory', {}).items(): + f.write(f"- **Connection Memory ({c} conns):** {mem:.2f} MB\n") + conn_keys = sorted([k for k in r.keys() if k.startswith("conn_storm_") and k.endswith("_output")], + key=lambda x: int(re.search(r'\d+', x).group(0))) + for conn_k in conn_keys: + c_val = re.search(r'\d+', conn_k).group(0) + tps_val = r.get(f"conn_storm_{c_val}_tps", -1) + p50_val = r.get(f"conn_storm_{c_val}_p50_ms", -1) + p99_val = r.get(f"conn_storm_{c_val}_p99_ms", -1) + if tps_val >= 0: + f.write(f"- **Connection Storm ({c_val} conns) Throughput:** {tps_val:.2f} conn/sec\n") + if p50_val >= 0: + f.write(f"- **Connection Storm ({c_val} conns) P50 Latency:** {p50_val:.2f} ms\n") + if p99_val >= 0: + f.write(f"- **Connection Storm ({c_val} conns) P99 Latency:** {p99_val:.2f} ms\n") + f.write(f"\n### Connection Storm ({c_val} conns) Performance\n```\n") + f.write(r[conn_k]) + f.write("\n```\n\n") + if "churn_tps" in r: + f.write(f"- **High Churn Throughput:** {r['churn_tps']:.2f} conn/sec\n") + f.write("### High Churn Performance\n```\n") + f.write(r.get('churn_output', '')) + f.write("\n```\n") + f.write(f"- **OS TIME_WAIT sockets count (post-churn):** {r.get('time_wait_count', -1)}\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 5. Query Engine & Indices\n") + if "suite_queries" in results: + r = results["suite_queries"] + f.write(f"- **Point Lookup REQ Time:** {r.get('query_point_time', -1):.2f} seconds\n") + f.write("### Point Lookup REQ Latencies\n```\n") + f.write(r.get('query_point_output', '')) + f.write("\n```\n") + f.write(f"- **Point Lookup COUNT (NIP-45) Time:** {r.get('query_point_count_time', -1):.2f} seconds\n") + f.write("### Point Lookup COUNT Latencies\n```\n") + f.write(r.get('query_point_count_output', '')) + f.write("\n```\n") + f.write(f"- **Complex Query REQ Time:** {r.get('query_complex_time', -1):.2f} seconds\n") + f.write("### Complex Query REQ Latencies\n```\n") + f.write(r.get('query_complex_output', '')) + f.write("\n```\n") + f.write(f"- **Complex COUNT (NIP-45) Query Time:** {r.get('query_complex_count_time', -1):.2f} seconds\n") + f.write("### Complex COUNT Latencies\n```\n") + f.write(r.get('query_complex_count_output', '')) + f.write("\n```\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 6. Active Monitors (Viral Post Fanout)\n") + if "suite_monitors" in results: + r = results["suite_monitors"] + f.write(f"- **Subscription Fan-out Time:** {r.get('monitor_fanout_time', -1):.2f} seconds\n") + f.write("### Fanout Output\n```\n") + f.write(r.get('monitor_output', '')) + f.write("\n```\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 7. Negentropy Sync\n") + if "suite_negentropy" in results: + r = results["suite_negentropy"] + f.write(f"- **Status:** {r.get('status', 'Pending')}\n") + if "elapsed" in r: + f.write(f"- **Sync Time:** {r['elapsed']:.2f} seconds\n") + f.write(f"- **Sync Throughput:** {r['tps']:.2f} events/sec\n") + f.write("\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 8. Write Policy Plugin\n") + if "suite_plugin" in results: + r = results["suite_plugin"] + f.write(f"- **Status:** {r.get('status', 'Pending')}\n") + if "elapsed" in r: + f.write(f"- **Plugin Ingestion Time:** {r['elapsed']:.2f} seconds\n") + f.write(f"- **Plugin Ingestion Throughput:** {r['tps']:.2f} events/sec\n") + f.write("\n### Plugin Ingestion Latencies\n```\n") + f.write(r.get('output', '')) + f.write("\n```\n") + f.write("\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + f.write("## 9 & 10. CLI & Dictionary Compression\n") + if "suite_cli" in results: + r = results["suite_cli"] + f.write(f"- **Import Time:** {r.get('import_time', -1):.2f} seconds\n") + f.write(f"- **Export Time:** {r.get('export_time', -1):.2f} seconds\n") + f.write(f"- **Dictionary Generation Time:** {r.get('dict_gen_time', -1):.2f} seconds\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 11. OS-Level Metrics\n") + if "suite_os" in results: + r = results["suite_os"] + f.write(f"- **Baseline RSS:** {r.get('process_rss', -1):.2f} MB\n") + f.write(f"- **CPU Utilization:** {r.get('cpu_utilization_pct', -1):.2f}%\n") + f.write(f"- **Write Amplification Factor (WAF):** {r.get('write_amplification_factor', -1):.2f}\n") + f.write(f"- **Bytes Written:** {r.get('write_bytes', -1) / (1024*1024):.2f} MB\n") + f.write(f"- **Bytes Read:** {r.get('read_bytes', -1) / (1024*1024):.2f} MB\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 12. Stress & Edge Cases\n") + if "suite_stress" in results: + r = results["suite_stress"] + f.write(f"- **Adversarial Tests:** Completed successfully\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + + f.write("## 13. Backpressure Performance\n") + if "suite_backpressure" in results: + r = results["suite_backpressure"] + f.write(f"- **Total Backpressure Test Time:** {r.get('backpressure_time', -1):.2f} seconds\n") + f.write("### Backpressure Latencies\n```\n") + f.write(r.get('backpressure_output', '')) + f.write("\n```\n\n") + else: + f.write("- **Status:** Skipped / Not Run\n\n") + +def main(): + parser = argparse.ArgumentParser(description="Strfry Performance Benchmarking Orchestrator") + parser.add_argument("--suite", type=str, help="Run a specific suite (e.g. storage, ingestion, stress)") + parser.add_argument("--skip-heavy", action="store_true", help="Skip heavy database generation to speed up testing") + parser.add_argument("--dry-run", action="store_true", help="Print commands but don't run tests") + args = parser.parse_args() + + manager = StrfryManager() + + suites_to_run = [] + all_suites = { + "storage": suite_storage, + "ingestion": suite_ingestion, + "concurrency": suite_concurrency, + "websockets": suite_websockets, + "queries": suite_queries, + "monitors": suite_monitors, + "negentropy": suite_negentropy, + "plugin": suite_plugin, + "cli": suite_cli_dict, + "os": suite_os, + "stress": suite_stress, + "backpressure": suite_backpressure + } + + if args.suite: + if args.suite in all_suites: + suites_to_run.append((args.suite, all_suites[args.suite])) + else: + print(f"[ERROR] Unknown suite: {args.suite}") + sys.exit(1) + elif args.skip_heavy: + suites_to_run = [(name, func) for name, func in all_suites.items() if name not in ["storage"]] + else: + suites_to_run = list(all_suites.items()) + + docker_available = check_docker() + if not docker_available: + print("[WARNING] Docker daemon is not running or not accessible. Docker-based out-of-core storage test will be skipped.") + + if args.dry_run: + print("[INFO] DRY RUN: Would execute the selected suites:") + for name, _ in suites_to_run: + print(f" - {name}") + return + + if any(name == "storage" for name, _ in suites_to_run): + manager.build_docker() + + print("[INFO] Pre-compiling strfry-bench...") + subprocess.run(["cargo", "build", "--release"], cwd=get_bench_dir(), check=True) + + results = {} + try: + for name, func in suites_to_run: + results[f"suite_{name}"] = func(manager, args.skip_heavy) + generate_report(results) + finally: + if any(name == "storage" for name, _ in suites_to_run): + print("[INFO] Purging leftover Docker resources to reclaim space...") + subprocess.run(["docker", "system", "prune", "-f"], stdout=subprocess.DEVNULL) + +if __name__ == "__main__": + main() + diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 00000000..7f58c059 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,135 @@ +# Strfry Benchmarking & Performance Profiling + +This document describes the strfry benchmarking framework, profiling methodologies, A/B comparison tools, and optimization analysis. + +--- + +## 1. Overview & Architecture + +The benchmarking infrastructure lives in the `bench/` directory and consists of: + +- **`bench/run_stats.py`**: The primary benchmarking orchestrator. It controls relay lifecycles, seeds databases, executes workload suites using the high-performance Rust benchmark client (`strfry-bench`), monitors system resources (`/proc/pid/status`, `/proc/pid/io`, `/proc/net/tcp`), and generates structured Markdown reports (`benchmark_report.md`). +- **`bench/ab_test.py`**: Relative A/B comparison runner. Stashes current branch changes, checks out the base branch (default: `master`), builds and benchmarks both branches under identical test parameters, collects hardware-agnostic metrics, and produces a delta report (`benchmark_comparison.md`). +- **`bench/alloc_tracker.c`**: Shared library (`LD_PRELOAD`) hooked via `dlsym` to record exact heap allocation counts and total bytes allocated during execution without modifying strfry binaries. +- **`strfry-bench`**: A multi-threaded Rust client designed to stress test Nostr relays via WebSocket connections (`ws://`), simulating events, NIP-45 counts, subscription fan-out, connection storms, high churn, and malicious traffic. + +### 1.1 Prerequisites & Requirements + +- **Linux `perf`**: Required for `perf stat` hardware instruction counting in A/B tests and `perf record` CPU profiling. Ensure unprivileged access is configured: + ```bash + sudo sysctl -w kernel.perf_event_paranoid=-1 + ``` +- **`inferno` (Flamegraph Generator)**: Required for collapsing stack traces and rendering interactive SVG flamegraphs: + ```bash + cargo install inferno + ``` +- **`gcc` & `make`**: Required for compiling strfry and `bench/alloc_tracker.c` for `LD_PRELOAD` heap allocation tracking. +- **Python 3**: Required to run `bench/run_stats.py` and `bench/ab_test.py` orchestrators. +- **Docker** *(Optional)*: Required only if running out-of-core memory limit benchmark suites. + + +## 2. Benchmark Suites + +The orchestrator executes 13 distinct benchmark suites: + +1. **Storage (In-Core vs Out-of-Core)**: Measures sequential scan throughput and pagination latencies when the database fits in memory (In-Core) vs under strict memory limits (256MB Docker container out-of-core pagination). +2. **Event Ingestion Pipeline**: Benchmarks write throughput (events/sec) for small (50B) payloads, large (10KB) payloads, and single-connection spam. +3. **Concurrency & Thread Pool**: Evaluates REQ query performance while heavy background write pressure is applied. +4. **WebSockets & Connections**: Tests connection establishment storms (100, 500, 5000 connections), memory footprint per connection (VmRSS), high connection churn rate, and OS `TIME_WAIT` socket cleanup. +5. **Query Engine & NIP-45 Indices**: Measures exact point lookup latency, NIP-45 `COUNT` performance, and complex multi-field NIP-01 filter queries. +6. **Active Monitors (Viral Post Fanout)**: Simulates subscription fan-out across dozens/hundreds of connected clients. +7. **Negentropy Sync**: Benchmarks NIP-77 set reconciliation synchronization speed and events/sec throughput between relays. +8. **Write Policy Plugin**: Evaluates IPC overhead when strfry delegates event validation to external policy plugins (`relay.writePolicy.plugin`). +9. **CLI & Dictionary Compression**: Measures `strfry import`, `strfry export`, and `strfry dict train` Zstd compression training performance. +10. **OS-Level Metrics**: Tracks baseline memory RSS, CPU utilization %, physical disk reads/writes, and Write Amplification Factor (WAF). +11. **Stress & Edge Cases**: Exercises adversarial workloads including Slow Loris connection holding and Signature Flood attacks. +12. **Backpressure Performance**: Tests event distribution when fast and slow WebSocket clients subscribe to the same feed. + +--- + +## 3. Running Benchmarks & A/B Comparisons + +### Running All Benchmark Suites + +```bash +python3 bench/run_stats.py +``` + +To skip the heavy 1-million-event database generation for faster local test runs: + +```bash +python3 bench/run_stats.py --skip-heavy +``` + +To run a single specific suite (e.g. `websockets`, `ingestion`, `queries`, `negentropy`): + +```bash +python3 bench/run_stats.py --suite websockets +``` + +### Running A/B Relative Comparison + +To compare your feature branch against `master`: + +```bash +python3 bench/ab_test.py --base master +``` + +To run A/B comparison with lightweight dataset sizes: + +```bash +python3 bench/ab_test.py --base master --skip-heavy +``` + +### Pre-PR Full Comparison + +Before opening a pull request, run the **full** A/B comparison to generate a definitive regression report covering all suites including the 1M-event storage test: + +```bash +python3 bench/ab_test.py --base master --full +``` + +The `--full` flag overrides `--skip-heavy` and ensures every suite runs with production-scale dataset sizes. The output report `benchmark_comparison.md` displays delta percentages (%) and status indicators: +- **Deterministic Metrics**: Retired CPU instructions (via `perf stat`), heap allocation count, and total bytes allocated (via `alloc_tracker.c`). +- **Wall-Clock & Resource Metrics**: Throughput, latencies (P50, P90, P95, P99), RSS memory growth, and disk WAF. + +--- + +## 4. Flamegraphs & Profiling + +Profiling strfry under heavy load helps pinpoint CPU bottlenecks and memory allocation hotspots. + +### Generating Flamegraphs with `perf` and `inferno` + +1. Install `inferno` (if not already installed): + ```bash + cargo install inferno + ``` + +2. Record profile with `perf`: + ```bash + perf record -o perf_import.data -F 99 -g -- ./strfry import --no-verify < bench/test_seed.jsonl + ``` + +3. Collapse stacks and render SVG flamegraph: + ```bash + perf script -i perf_import.data | inferno-collapse-perf | inferno-flamegraph > flamegraph_import.svg + ``` + +--- + +## 5. Identified Hotspots & Optimization Opportunities + +Profile data gathered from `perf stat`, flamegraphs, and allocation tracking highlights key areas: + +1. **Negentropy Build Transaction Windowing (Resolved in PR)**: + - *Problem*: `strfry negentropy build` previously held a single write transaction (`txn_rw`) across the entire DB scan and tree construction phase, blocking concurrent relay writes. + - *Optimization*: Refactored to scan matching events in read-only batches (`txn_ro`) of 10,000 events, followed by short write transaction windows (`txn_rw`). This reduced lock contention while improving negentropy sync throughput. + +2. **JSON AST Allocation Overhead**: + - *Hotspot*: `tao::json` creates heap-allocated C++ `variant` structures for every parsed JSON key, value, and tag array during event ingestion. + - *Opportunity*: Streaming or zero-copy JSON parsing directly into `PackedEventBuilder` avoids intermediate `basic_value` DOM allocations during high-rate ingestion. + +3. **LMDB Secondary Index Insertion**: + - *Hotspot*: Writing secondary index entries (`Event__pubkey`, `Event__created_at`, `Event__kind`, `Event__tag`) during bulk import incurs CPU overhead in string comparison (`lmdb_comparator__StringUint64`). + - *Opportunity*: Pre-sorting events or index keys prior to insertion minimizes LMDB page splitting during initial bulk imports. diff --git a/golpe b/golpe index 72477956..2ca73fe6 160000 --- a/golpe +++ b/golpe @@ -1 +1 @@ -Subproject commit 72477956b080bbb957118f8843ee99d7153eff3a +Subproject commit 2ca73fe6c0617c0a80f5093e424cfa0031baa17b diff --git a/src/apps/dbutils/cmd_negentropy.cpp b/src/apps/dbutils/cmd_negentropy.cpp index 9293d93b..ad269b93 100644 --- a/src/apps/dbutils/cmd_negentropy.cpp +++ b/src/apps/dbutils/cmd_negentropy.cpp @@ -73,53 +73,62 @@ void cmd_negentropy(const std::vector &subArgs) { } else if (args["build"].asBool()) { uint64_t treeId = args[""].asLong(); + std::string filterStr; + { + auto txn = env.txn_ro(); + auto view = env.lookup_NegentropyFilter(txn, treeId); + if (!view) throw herr("couldn't find treeId: ", treeId); + filterStr = view->filter(); + } + + NostrFilter f(tao::json::from_string(filterStr), MAX_U64); + DBScan scanner(f); + struct Record { uint64_t created_at; Bytes32 id; }; std::vector recs; + recs.reserve(10000); - // Read-only phase: fetch filter and collect matching events without - // blocking writers. - { - auto txn = env.txn_ro(); - - std::string filterStr; - + while (1) { + // Read a batch of up to 10,000 matching events using a read-only transaction. + // This prevents blocking other database writers during the scan phase. { - auto view = env.lookup_NegentropyFilter(txn, treeId); - if (!view) throw herr("couldn't find treeId: ", treeId); - filterStr = view->filter(); - } - - DBQuery query(tao::json::from_string(filterStr)); - - while (1) { - bool complete = query.process(txn, [&](const auto &sub, uint64_t levId){ + auto txn = env.txn_ro(); + scanner.scan(txn, [&](uint64_t levId) { auto ev = lookupEventByLevId(txn, levId); auto packed = PackedEventView(ev.buf); recs.emplace_back(packed.created_at(), packed.id()); - }); - - if (complete) break; + return recs.size() >= 10000; + }, [](uint64_t) { return false; }); } - } - - // Write phase: store collected records into the negentropy BTree. - { - auto txn = env.txn_rw(); - increaseModCounter(txn); - negentropy::storage::BTreeLMDB storage(txn, negentropyDbi, treeId); + if (recs.empty()) { + break; + } - for (const auto &r : recs) { - storage.insert(r.created_at, r.id.sv()); + // Write the batch of events to the negentropy BTree. + // Holding the write lock only during BTree insertion keeps write lock windows short. + { + auto txn = env.txn_rw(); + increaseModCounter(txn); + + negentropy::storage::BTreeLMDB storage(txn, negentropyDbi, treeId); + for (const auto &r : recs) { + storage.insert(r.created_at, r.id.sv()); + } + storage.flush(); + txn.commit(); } - storage.flush(); + // If we fetched less than 10,000, we've scanned all matching events. + if (recs.size() < 10000) { + break; + } - txn.commit(); + recs.clear(); } } } diff --git a/test/runTests.sh b/test/runTests.sh index 7cc6a991..62b7ce3c 100755 --- a/test/runTests.sh +++ b/test/runTests.sh @@ -52,4 +52,7 @@ perl "./test/tests/runSyncTests.pl" \ && pass "./test/tests/syncTests.pl" \ || fail "./test/tests/runSyncTests.pl failed" +info "cleaning up test databases..." +rm -rf "strfry-db-test" "strfry-db-test-1" "strfry-db-test-2" + pass "All tests passed." diff --git a/test/tests/filterFuzzTest.pl b/test/tests/filterFuzzTest.pl index 5e27447a..4f393660 100755 --- a/test/tests/filterFuzzTest.pl +++ b/test/tests/filterFuzzTest.pl @@ -2,7 +2,7 @@ use strict; use Data::Dumper; -use JSON::XS; +use JSON::PP; use IPC::Open2; # ./strfry export|perl -MJSON::XS -nE '$z=decode_json($_); for my $t (@{$z->{tags}}) { say $t->[1] if $t->[0] eq "e"}'|sort|uniq -c|sort -rn|head -50|perl -nE '/\d+\s+(\w+)/ && say $1' diff --git a/test/utils/dumbFilter.pl b/test/utils/dumbFilter.pl index 1aefbec1..3fb194a7 100755 --- a/test/utils/dumbFilter.pl +++ b/test/utils/dumbFilter.pl @@ -2,7 +2,7 @@ use strict; -use JSON::XS; +use JSON::PP; binmode(STDOUT, ":utf8"); diff --git a/ubuntu.Dockerfile b/ubuntu.Dockerfile index 0af0fb05..64a6cd40 100644 --- a/ubuntu.Dockerfile +++ b/ubuntu.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Create a non-root user for security RUN useradd -m -d /app -s /bin/bash strfry && \ + mkdir -p /app/strfry-db && \ chown -R strfry:strfry /app # Switch to the unprivileged user @@ -39,7 +40,6 @@ USER strfry COPY --from=build --chown=strfry:strfry /build/strfry /app/strfry COPY --from=build --chown=strfry:strfry /build/strfry.conf /app/strfry.conf -COPY --from=build --chown=strfry:strfry /build/strfry-db /app/strfry-db EXPOSE 7777