diff --git a/scripts/capture_req_results.py b/scripts/capture_req_results.py new file mode 100755 index 00000000..81af61f6 --- /dev/null +++ b/scripts/capture_req_results.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import json +import time +import websockets + + +async def issue_req(relay_url: str, filter_dict: dict, timeout: float = 30.0) -> dict: + try: + async with websockets.connect(relay_url) as ws: + sub_id = f"test-{int(time.time()*1000)}" + req_msg = json.dumps(["REQ", sub_id, filter_dict]) + await ws.send(req_msg) + events = [] + eose_received = False + start_time = time.time() + while not eose_received: + remaining = timeout - (time.time() - start_time) + if remaining <= 0: + break + + try: + msg = await asyncio.wait_for(ws.recv(), timeout=remaining) + data = json.loads(msg) + if len(data) >= 2: + msg_type = data[0] + if msg_type == "EVENT" and len(data) >= 3: + events.append(data[2]) + elif msg_type == "EOSE": + eose_received = True + except asyncio.TimeoutError: + break + + return { + "filter": filter_dict, + "event_count": len(events), + "event_ids": [e.get("id") for e in events if "id" in e], + "events": events, + "eose_received": eose_received, + } + except Exception as e: + return { + "filter": filter_dict, + "error": str(e), + "event_count": 0, + "event_ids": [], + "events": [], + "eose_received": False, + } + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--relay", default="ws://localhost:7777") + parser.add_argument("--filter", action="append") + parser.add_argument("--output", default="req_results.json") + + args = parser.parse_args() + if not args.filter: + args.filter = ['{"kinds":[1],"limit":500}'] + + results = [] + for i, filter_str in enumerate(args.filter): + try: + filter_dict = json.loads(filter_str) + except json.JSONDecodeError: + continue + result = await issue_req(args.relay, filter_dict) + results.append(result) + + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/compare_results.py b/scripts/compare_results.py new file mode 100755 index 00000000..fa1881a6 --- /dev/null +++ b/scripts/compare_results.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import sys +import json + + +def load_results(path: str) -> list: + with open(path) as f: + return json.load(f) + + +def extract_event_ids(results: list) -> dict: + id_sets = {} + for i, result in enumerate(results): + filter_key = json.dumps(result.get("filter", {})) + event_ids = set(result.get("event_ids", [])) + id_sets[filter_key] = event_ids + return id_sets + + +def compare(baseline_results: list, patched_results: list) -> bool: + baseline_ids = extract_event_ids(baseline_results) + patched_ids = extract_event_ids(patched_results) + + all_match = True + for filter_key in baseline_ids: + baseline_set = baseline_ids.get(filter_key, set()) + patched_set = patched_ids.get(filter_key, set()) + filter_obj = json.loads(filter_key) + + if baseline_set != patched_set: + all_match = False + missing = baseline_set - patched_set + extra = patched_set - baseline_set + print(f"Mismatch: filter {filter_obj}") + if missing: + print(f" Missing: {len(missing)}") + if extra: + print(f" Extra: {len(extra)}") + + return all_match + + +def main(): + if len(sys.argv) != 3: + return + baseline = load_results(sys.argv[1]) + patched = load_results(sys.argv[2]) + sys.exit(0 if compare(baseline, patched) else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/mixed_load_bench.py b/scripts/mixed_load_bench.py new file mode 100755 index 00000000..96842f0e --- /dev/null +++ b/scripts/mixed_load_bench.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import json +import time +import random +import string +from dataclasses import dataclass, field +import websockets + + +@dataclass +class BenchmarkResults: + write_rate_target: int + write_rate_actual: float = 0.0 + read_rate_actual: float = 0.0 + req_p50_ms: float = 0.0 + req_p95_ms: float = 0.0 + req_p99_ms: float = 0.0 + req_latencies_ms: list = field(default_factory=list) + total_events_sent: int = 0 + total_requests: int = 0 + errors: int = 0 + + +def generate_random_key() -> str: + return ''.join(random.choices(string.hexdigits[:16], k=64)) + + +def generate_event(kind: int = 1, author: str = "") -> dict: + if not author: + author = generate_random_key() + created_at = int(time.time()) + + event = { + "kind": kind, + "pubkey": author, + "created_at": created_at, + "tags": [], + "content": f"Benchmark event {random.randint(1000, 9999)}", + "sig": generate_random_key()[:128], + "id": generate_random_key(), + } + return event + + +async def writer_task(relay_url: str, write_interval: float, duration: float, results: BenchmarkResults): + start_time = time.time() + try: + async with websockets.connect(relay_url) as ws: + while time.time() - start_time < duration: + try: + event = generate_event() + msg = json.dumps(["EVENT", event]) + await ws.send(msg) + try: + await asyncio.wait_for(ws.recv(), timeout=0.5) + except asyncio.TimeoutError: + pass + + results.total_events_sent += 1 + await asyncio.sleep(write_interval) + except Exception as e: + results.errors += 1 + await asyncio.sleep(0.1) + except Exception as e: + results.errors += 1 + + +async def reader_task(relay_url: str, req_interval: float, duration: float, results: BenchmarkResults): + start_time = time.time() + try: + async with websockets.connect(relay_url) as ws: + reader_id = random.randint(0, 9999) + while time.time() - start_time < duration: + try: + sub_id = f"reader-{reader_id}-{int(time.time()*1000)}" + filter_dict = {"kinds": [1], "limit": 500} + req_msg = json.dumps(["REQ", sub_id, filter_dict]) + + req_time = time.time() + await ws.send(req_msg) + eose_received = False + while not eose_received: + try: + msg = await asyncio.wait_for(ws.recv(), timeout=30.0) + data = json.loads(msg) + if len(data) >= 2 and data[0] == "EOSE": + eose_received = True + except asyncio.TimeoutError: + break + + latency_ms = (time.time() - req_time) * 1000 + results.req_latencies_ms.append(latency_ms) + results.total_requests += 1 + + await asyncio.sleep(req_interval) + except Exception as e: + results.errors += 1 + await asyncio.sleep(0.1) + except Exception as e: + results.errors += 1 + + +def percentile(data: list, p: int) -> float: + if not data: + return 0.0 + sorted_data = sorted(data) + index = int(len(sorted_data) * p / 100) + return sorted_data[min(index, len(sorted_data) - 1)] + + +async def run_benchmark(relay_url: str, write_rate: int, read_rate: int, + writers: int, readers: int, duration: float) -> BenchmarkResults: + results = BenchmarkResults(write_rate_target=write_rate) + write_interval = writers / max(write_rate, 1) + read_interval = readers / max(read_rate, 1) + tasks = [] + for _ in range(writers): + tasks.append(writer_task(relay_url, write_interval, duration, results)) + for _ in range(readers): + tasks.append(reader_task(relay_url, read_interval, duration, results)) + + await asyncio.gather(*tasks) + + elapsed = duration + results.write_rate_actual = results.total_events_sent / elapsed + results.read_rate_actual = results.total_requests / elapsed + + if results.req_latencies_ms: + results.req_p50_ms = percentile(results.req_latencies_ms, 50) + results.req_p95_ms = percentile(results.req_latencies_ms, 95) + results.req_p99_ms = percentile(results.req_latencies_ms, 99) + + return results + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--relay", default="ws://localhost:7777") + parser.add_argument("--write-rate", type=int, default=1000) + parser.add_argument("--read-rate", type=int, default=10) + parser.add_argument("--writers", type=int, default=10) + parser.add_argument("--readers", type=int, default=50) + parser.add_argument("--duration", type=float, default=60) + parser.add_argument("--output", default="bench_results.json") + + args = parser.parse_args() + try: + results = await run_benchmark( + args.relay, + args.write_rate, + args.read_rate, + args.writers, + args.readers, + args.duration + ) + + print(f"Write: {results.write_rate_actual:.1f} ev/s") + print(f"Read: {results.read_rate_actual:.1f} REQ/s") + print(f"p50={results.req_p50_ms:.1f}ms p95={results.req_p95_ms:.1f}ms p99={results.req_p99_ms:.1f}ms") + + output_dict = { + "write_rate_target": results.write_rate_target, + "write_rate_actual": results.write_rate_actual, + "read_rate_actual": results.read_rate_actual, + "req_p50_ms": results.req_p50_ms, + "req_p95_ms": results.req_p95_ms, + "req_p99_ms": results.req_p99_ms, + "total_events_sent": results.total_events_sent, + "total_requests": results.total_requests, + "errors": results.errors, + } + + with open(args.output, "w") as f: + json.dump(output_dict, f, indent=2) + print(f"Saved to {args.output}") + except Exception as e: + print(f"Error: {e}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/parse_scan_metrics.py b/scripts/parse_scan_metrics.py new file mode 100755 index 00000000..74492fb4 --- /dev/null +++ b/scripts/parse_scan_metrics.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +import re +import sys +import json +from collections import defaultdict +from dataclasses import dataclass +from typing import List + +LOG_RE = re.compile( + r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?' + r"REQ='([^']+)'\s+" + r'scan=(\w+)\s+' + r'indexOnly=(\d+)\s+' + r'time=(\d+)us\s+' + r'saveRestores=(\d+)\s+' + r'recsFound=(\d+)\s+' + r'work=(\d+)' +) + +@dataclass +class ScanRecord: + timestamp: str + sub_id: str + scan_type: str + index_only: bool + time_us: int + save_restores: int + recs_found: int + work: int + +def parse_log(path: str) -> List[ScanRecord]: + records = [] + with open(path) as f: + for line in f: + m = LOG_RE.search(line) + if m: + records.append(ScanRecord( + timestamp=m.group(1), + sub_id=m.group(2), + scan_type=m.group(3), + index_only=bool(int(m.group(4))), + time_us=int(m.group(5)), + save_restores=int(m.group(6)), + recs_found=int(m.group(7)), + work=int(m.group(8)), + )) + return records + +def report(records: List[ScanRecord]): + if not records: + print("No scan metric lines found. Is logScanMetrics enabled?") + return + + paused = [r for r in records if r.save_restores > 0] + by_type = defaultdict(list) + for r in records: + by_type[r.scan_type].append(r) + + print(f"Total: {len(records)}") + print(f"Paused: {len(paused)} ({100*len(paused)/len(records):.1f}%)") + + if paused: + print(f"Max restores: {max(r.save_restores for r in records)}") + print(f"Paused avg: {sum(r.time_us for r in paused)/len(paused):.0f}µs") + + unpaused = [r for r in records if r.save_restores == 0] + if unpaused: + print(f"Unpaused avg: {sum(r.time_us for r in unpaused)/len(unpaused):.0f}µs") + + print(f"\n{'Type':<10} {'Count':>6} {'Paused':>6} {'AvgTime':>10} {'AvgRest':>8}") + for scan_type, recs in sorted(by_type.items()): + paused_recs = [r for r in recs if r.save_restores > 0] + avg_time = sum(r.time_us for r in recs) / len(recs) + avg_restores = sum(r.save_restores for r in recs) / len(recs) + print(f"{scan_type:<10} {len(recs):>6} {len(paused_recs):>6} " + f"{avg_time:>10.0f} {avg_restores:>8.1f}") + +if __name__ == "__main__": + path = sys.argv[1] if len(sys.argv) > 1 else "relay.log" + records = parse_log(path) + report(records) + with open("scan_metrics.json", "w") as f: + json.dump([vars(r) for r in records], f, indent=2) diff --git a/scripts/populate_test_db.py b/scripts/populate_test_db.py new file mode 100755 index 00000000..6d7155eb --- /dev/null +++ b/scripts/populate_test_db.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import json +import time +import random +import string +from typing import Optional +import websockets + + +def generate_random_key() -> str: + return ''.join(random.choices(string.hexdigits[:16], k=64)) + + +def generate_event(kind: int, author: Optional[str] = None, content: str = "") -> dict: + if author is None: + author = generate_random_key() + created_at = int(time.time()) - random.randint(0, 86400) + + event = { + "kind": kind, + "pubkey": author, + "created_at": created_at, + "tags": [], + "content": content or f"Test event {random.randint(1000, 9999)}", + } + event["sig"] = generate_random_key()[:128] + event["id"] = generate_random_key() + + return event + + +async def populate_relay(relay_url: str, num_events: int, kinds: list[int]): + try: + async with websockets.connect(relay_url) as ws: + sent = 0 + authors = [generate_random_key() for _ in range(20)] + + for i in range(num_events): + kind = random.choice(kinds) + author = random.choice(authors) + event = generate_event(kind, author, f"Content #{i}") + msg = json.dumps(["EVENT", event]) + try: + await ws.send(msg) + try: + response = await asyncio.wait_for(ws.recv(), timeout=1.0) + resp = json.loads(response) + if resp[0] == "OK": + sent += 1 + except asyncio.TimeoutError: + sent += 1 + except Exception as e: + break + print(f"Sent {sent}/{num_events}") + return sent + except Exception as e: + print(f"Error: {e}") + return 0 + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--relay", default="ws://localhost:7777") + parser.add_argument("--events", type=int, default=1000) + parser.add_argument("--kinds", default="1,0,3") + + args = parser.parse_args() + kinds = [int(k) for k in args.kinds.split(",")] + await populate_relay(args.relay, args.events, kinds) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/write_rate_sweep.py b/scripts/write_rate_sweep.py new file mode 100755 index 00000000..037d4817 --- /dev/null +++ b/scripts/write_rate_sweep.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import json +import time +import random +import string +from dataclasses import dataclass, field +import websockets + + +@dataclass +class BenchmarkResults: + write_rate_target: int + write_rate_actual: float = 0.0 + read_rate_actual: float = 0.0 + req_p50_ms: float = 0.0 + req_p95_ms: float = 0.0 + req_p99_ms: float = 0.0 + req_latencies_ms: list = field(default_factory=list) + total_events_sent: int = 0 + total_requests: int = 0 + errors: int = 0 + + +def generate_random_key() -> str: + return ''.join(random.choices(string.hexdigits[:16], k=64)) + + +def generate_event(kind: int = 1, author: str = "") -> dict: + if not author: + author = generate_random_key() + + created_at = int(time.time()) + + event = { + "kind": kind, + "pubkey": author, + "created_at": created_at, + "tags": [], + "content": f"Benchmark event {random.randint(1000, 9999)}", + "sig": generate_random_key()[:128], + "id": generate_random_key(), + } + return event + + +async def writer_task(relay_url: str, write_interval: float, duration: float, results: BenchmarkResults): + start_time = time.time() + try: + async with websockets.connect(relay_url) as ws: + while time.time() - start_time < duration: + try: + event = generate_event() + msg = json.dumps(["EVENT", event]) + await ws.send(msg) + try: + await asyncio.wait_for(ws.recv(), timeout=0.5) + except asyncio.TimeoutError: + pass + + results.total_events_sent += 1 + await asyncio.sleep(write_interval) + except Exception as e: + results.errors += 1 + await asyncio.sleep(0.1) + except Exception as e: + results.errors += 1 + + +async def reader_task(relay_url: str, req_interval: float, duration: float, results: BenchmarkResults): + start_time = time.time() + try: + async with websockets.connect(relay_url) as ws: + reader_id = random.randint(0, 9999) + while time.time() - start_time < duration: + try: + sub_id = f"reader-{reader_id}-{int(time.time()*1000)}" + filter_dict = {"kinds": [1], "limit": 500} + req_msg = json.dumps(["REQ", sub_id, filter_dict]) + + req_time = time.time() + await ws.send(req_msg) + eose_received = False + while not eose_received: + try: + msg = await asyncio.wait_for(ws.recv(), timeout=30.0) + data = json.loads(msg) + if len(data) >= 2 and data[0] == "EOSE": + eose_received = True + except asyncio.TimeoutError: + break + + latency_ms = (time.time() - req_time) * 1000 + results.req_latencies_ms.append(latency_ms) + results.total_requests += 1 + + await asyncio.sleep(req_interval) + except Exception as e: + results.errors += 1 + await asyncio.sleep(0.1) + except Exception as e: + results.errors += 1 + + +def percentile(data: list, p: int) -> float: + if not data: + return 0.0 + sorted_data = sorted(data) + index = int(len(sorted_data) * p / 100) + return sorted_data[min(index, len(sorted_data) - 1)] + + +async def run_benchmark(relay_url: str, write_rate: int, read_rate: int, + writers: int, readers: int, duration: float) -> BenchmarkResults: + results = BenchmarkResults(write_rate_target=write_rate) + + write_interval = writers / max(write_rate, 1) + read_interval = readers / max(read_rate, 1) + write_interval = max(write_interval, 0.001) + read_interval = max(read_interval, 0.001) + + tasks = [] + for _ in range(writers): + tasks.append(writer_task(relay_url, write_interval, duration, results)) + for _ in range(readers): + tasks.append(reader_task(relay_url, read_interval, duration, results)) + await asyncio.gather(*tasks) + elapsed = duration + results.write_rate_actual = results.total_events_sent / elapsed + results.read_rate_actual = results.total_requests / elapsed + + if results.req_latencies_ms: + results.req_p50_ms = percentile(results.req_latencies_ms, 50) + results.req_p95_ms = percentile(results.req_latencies_ms, 95) + results.req_p99_ms = percentile(results.req_latencies_ms, 99) + + return results + + +async def main(): + parser = argparse.ArgumentParser(description="Write-rate sweep") + parser.add_argument("--relay", default="ws://localhost:7777", help="Relay URL") + parser.add_argument("--rates", default="0,100,500,1000,2000,5000", + help="Comma-separated write rates to test") + parser.add_argument("--duration", type=float, default=90, help="Duration per sweep point") + parser.add_argument("--output", default="sweep_results.json", help="Output file") + + args = parser.parse_args() + rates = [int(r) for r in args.rates.split(",")] + print(f"Write-rate sweep starting") + print(f"Relay: {args.relay}") + print(f"Write rates: {rates}") + print(f"Duration per point: {args.duration}s\n") + + results = [] + + for i, rate in enumerate(rates): + print(f"[{i+1}/{len(rates)}] write_rate={rate}") + result = await run_benchmark(args.relay, rate, 10, 10, 50, args.duration) + + results.append({ + "write_rate_target": result.write_rate_target, + "write_rate_actual": result.write_rate_actual, + "read_rate_actual": result.read_rate_actual, + "req_p50_ms": result.req_p50_ms, + "req_p95_ms": result.req_p95_ms, + "req_p99_ms": result.req_p99_ms, + "total_events_sent": result.total_events_sent, + "total_requests": result.total_requests, + "errors": result.errors, + }) + + print(f" p50={result.req_p50_ms:.1f} p95={result.req_p95_ms:.1f} p99={result.req_p99_ms:.1f} actual={result.write_rate_actual:.1f}") + if i < len(rates) - 1: + await asyncio.sleep(5) + + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + print(f"\nResults saved to {args.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/DBQuery.h b/src/DBQuery.h index 7e91f4bd..649580f3 100644 --- a/src/DBQuery.h +++ b/src/DBQuery.h @@ -101,6 +101,9 @@ struct DBScan : NonCopyable { uint64_t nextInitIndex = 0; uint64_t approxWork = 0; + std::deque refillBuffer; + std::deque mergeBuffer; + DBScan(const NostrFilter &f) : f(f) { indexOnly = f.indexOnlyScans; @@ -268,12 +271,13 @@ struct DBScan : NonCopyable { cursors[ev.scanIndex()].outstanding--; if (cursors[ev.scanIndex()].outstanding == 0) { - std::deque moreEvents; - std::deque newEventQueue; - approxWork += cursors[ev.scanIndex()].collect(txn, *this, ev.scanIndex(), refillScanDepth, moreEvents); + refillBuffer.clear(); + mergeBuffer.clear(); + + approxWork += cursors[ev.scanIndex()].collect(txn, *this, ev.scanIndex(), refillScanDepth, refillBuffer); - std::merge(eventQueue.begin(), eventQueue.end(), moreEvents.begin(), moreEvents.end(), std::back_inserter(newEventQueue), cmp); - eventQueue.swap(newEventQueue); + std::merge(eventQueue.begin(), eventQueue.end(), refillBuffer.begin(), refillBuffer.end(), std::back_inserter(mergeBuffer), cmp); + eventQueue.swap(mergeBuffer); } } }