From 15af0ffd87d6a3c805afaeace5d3a29fdadda0e4 Mon Sep 17 00:00:00 2001 From: Ruslan Dautkhanov Date: Fri, 22 May 2026 08:29:59 -0600 Subject: [PATCH] perf: add 4 CodSpeed scenarios for cold-start work (issue #557) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each scenario maps 1:1 to a planned cold-start improvement so a per- PR CodSpeed delta is unambiguous. Without these, the wins from the upcoming PRs would be invisible to the dashboard. * XA-3 — attribute_walk_jvm_java_lang_System 1 000 walks of `gateway.jvm.java.lang.System`. Each walk re-issues 3 reflection round-trips because JVMView / JavaPackage / JavaClass __getattr__ don't memoize today. Cache canary for the upcoming Python attribute-cache PR — should collapse to ~0 round-trips after first walk once memoization lands. * XB — gateway_reconnect_against_running_jvm 50 fresh JavaGateway() instances against the fixture JVM. Measures per-connection setup cost: Java accept() loop allocates 14 command class instances per connection (GatewayConnection.java:196-208), Python runs _create_connection + socket setup. Lever for the Java-side command-prototype-cache PR. * XC — callback_infra_overhead_unused X1-1 workload (10 000 currentTimeMillis() calls) with CallbackServer started but no callbacks invoked. Delta vs X1-1 is the steady-state overhead of running the callback server alongside the main gateway. Target of the lazy-CallbackClient PR. * XD — full_cold_start_subprocess_first_call subprocess.Popen -> first call -> shutdown, one cycle per round. Slow per-iteration (~350 ms on M-series + JDK 21, multi-second on slower hosts); CodSpeed adapts iteration count. Target of the JVM-flag opt-ins (TieredStopAtLevel=1, AppCDS, CRaC). Implementation notes: XA / XB / XC follow the existing MacroScenario contract and run through the macro_gateway fixture in codspeed_macros.py (shared JVM, setup once, measure many). XD owns its JVM lifecycle inside measure() — it must NOT share the fresh_jvm()-managed fixture because both default to port 25333. So XD is registered in a new _COLD_START_SCENARIOS list parametrized through a new test_cold_start_scenario function. XD is also omitted from ALL_MACRO_CLASSES so the framework runner doesn't try to share its JVM either. Local measurements on M-series + JDK 21 (Temurin 21.0.10): XA-3: ~94 ms / 1000 walks = 94 us per walk (3 RTTs each) XB: ~21 ms / 50 reconnects = 425 us per reconnect XC: ~236 ms / 10k calls = 23.6 us per call XD: ~355 ms per full cold-start cycle Co-authored-by: Isaac --- py4j-python/src/py4j/tests/perf/README.md | 6 + .../tests/perf/scenarios/codspeed_macros.py | 72 +++++++++- .../src/py4j/tests/perf/scenarios/macro.py | 133 ++++++++++++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) diff --git a/py4j-python/src/py4j/tests/perf/README.md b/py4j-python/src/py4j/tests/perf/README.md index 44805ec5..197244f4 100644 --- a/py4j-python/src/py4j/tests/perf/README.md +++ b/py4j-python/src/py4j/tests/perf/README.md @@ -142,6 +142,12 @@ Aggregate throughput. Scenario IDs with variants reveal scaling curves. | X4 | `Collections.sort` on Python-`Comparable` proxies | Callback round-trip | | X5 | 1 000 × `ArrayList.get(-1)` → `Py4JJavaError` | Error-path latency | | X6 | 50 concurrent threads, 20 calls each | Pool saturation tail latency | +| X7-1k / X7-16k / X7-256k | `ByteBuffer.array()` returning N bytes | Bytes recv (decode_bytearray) | +| X8-1k / X8-16k / X8-256k | `BAOS.write(bytes, 0, N)` of N bytes | Bytes send (encode_bytearray) | +| XA-3 | 1 000 walks of `gateway.jvm.java.lang.System` | Attribute-resolution cache canary (issue #557) | +| XB | 50 fresh `JavaGateway()` against a running JVM | Per-connection setup cost (issue #557) | +| XC | X1-1 workload with CallbackServer started, no callbacks invoked | Callback-infra overhead delta (issue #557) | +| XD | `subprocess.Popen` → first call → shutdown, one cycle per round | Full cold start (issue #557 — CodSpeed only) | ## How to read the report diff --git a/py4j-python/src/py4j/tests/perf/scenarios/codspeed_macros.py b/py4j-python/src/py4j/tests/perf/scenarios/codspeed_macros.py index 0f18d494..8a74d1e4 100644 --- a/py4j-python/src/py4j/tests/perf/scenarios/codspeed_macros.py +++ b/py4j-python/src/py4j/tests/perf/scenarios/codspeed_macros.py @@ -11,8 +11,13 @@ on the dashboard, so regressions show up per-scenario rather than as one opaque aggregate. -Scenario coverage rationale: four macros covering the perf -characteristics most prone to regression on the py4j socket / call path: +Scenario coverage rationale: macros covering the perf characteristics +most prone to regression on the py4j socket / call path. Two test +functions live here: + +``test_macro_scenario`` — scenarios that share a long-running JVM +provided by the ``macro_gateway`` fixture. Each scenario reuses the +same gateway across rounds (setup once, measure many). * ``X1-1`` — single-thread concurrent_1_thread (10k sequential calls). Baseline round-trip latency floor. @@ -23,6 +28,18 @@ the most complex code path in py4j. * ``X6`` — pool_saturation_50_threads. Connection pool behavior under high concurrency, tail-latency sensitive. +* ``X7-16k`` / ``X8-16k`` — bytes recv/send 16k. +* ``XA-3`` — attribute walk depth (Python __getattr__ cache canary). +* ``XB`` — gateway reconnect against running JVM (per-connection + setup cost). +* ``XC`` — callback infra overhead unused (delta vs X1-1). + +``test_cold_start_scenario`` — scenarios that own their full JVM +lifecycle inside ``measure()``. Cannot share the fixture's JVM +because port 25333 is already in use; each round must spawn and +shut down its own subprocess. + +* ``XD`` — full_cold_start_subprocess_first_call. Adding more scenarios later is one parametrize entry per scenario. """ @@ -41,6 +58,10 @@ X6_PoolSaturation, X7_16k, X8_16k, + XA_AttributeWalk3, + XB_GatewayReconnect, + XC_CallbackInfraOverheadUnused, + XD_FullColdStart, ) @@ -51,9 +72,29 @@ # invisible to the per-PR dashboard — every prior macro returns # int / list / void / callback and the byte path was a measurement # blind spot. +# +# XA-XC add cold-start / attribute-walk / connection-setup / callback- +# infra measurement surfaces that share the long-running fixture JVM. +# Each maps 1:1 to a planned cold-start improvement PR (issue #557): +# XA — attribute caches (Python __getattr__ memoization) +# XB — Java per-connection command-prototype cache + background warm +# XC — lazy CallbackClient (delta vs X1-1 = current overhead) +# XD lives in _COLD_START_SCENARIOS below — owns its JVM lifecycle. +# Without these, a per-PR CodSpeed dashboard would not see the wins. _MACRO_SCENARIOS = [ X1_1Thread, X2_10k, X4_Callbacks, X6_PoolSaturation, X7_16k, X8_16k, + XA_AttributeWalk3, XB_GatewayReconnect, + XC_CallbackInfraOverheadUnused, +] + +# Scenarios whose measure() spawns its own JVM subprocess per round. +# They must NOT share the fixture's JVM — the fixture's JVM is already +# bound to the default port 25333, so a second spawn would race on +# bind. Each cold-start scenario is self-contained: spawn, connect, +# call, shut down. +_COLD_START_SCENARIOS = [ + XD_FullColdStart, ] @@ -94,3 +135,30 @@ def test_macro_scenario(benchmark, macro_gateway): if hasattr(scenario, "setup"): scenario.setup(gateway) benchmark(scenario.measure, gateway) + + +@pytest.mark.parametrize( + "scenario_cls", _COLD_START_SCENARIOS, + ids=[cls.id for cls in _COLD_START_SCENARIOS], +) +def test_cold_start_scenario(benchmark, scenario_cls): + """Run a cold-start scenario that owns its JVM lifecycle. + + No shared fixture JVM — each ``measure()`` invocation spawns a + fresh subprocess, runs one full cold-start cycle, and tears it + down. Slow per-iteration (~300 ms on M-series + JDK 21, multiple + seconds on slower hosts); CodSpeed adapts iteration count + accordingly. + + Skips cleanly if the Java side hasn't been built — checked via the + same ``verify_classpath`` path used by ``fresh_jvm``. + """ + from py4j.tests.perf.jvm import verify_classpath + try: + verify_classpath() + except JvmNotBuiltError as e: + pytest.skip(str(e)) + scenario = scenario_cls() + # Cold-start scenarios have no shared state; setup() is not + # expected. measure() ignores the gateway arg (it owns its own). + benchmark(scenario.measure, None) diff --git a/py4j-python/src/py4j/tests/perf/scenarios/macro.py b/py4j-python/src/py4j/tests/perf/scenarios/macro.py index 192840ee..c37fcb92 100644 --- a/py4j-python/src/py4j/tests/perf/scenarios/macro.py +++ b/py4j-python/src/py4j/tests/perf/scenarios/macro.py @@ -9,6 +9,7 @@ import threading +from py4j.java_gateway import JavaGateway from py4j.tests.perf.runner import MacroScenario @@ -380,6 +381,129 @@ class X8_256k(_X8BytesSendBase): iterations_per_round = 25 +# ===================================================================== XA +# Attribute walk depth: re-walks the gateway.jvm.java.lang.System +# attribute chain many times. py4j's JVMView / JavaPackage / JavaClass +# __getattr__ methods do NOT memoize their results today, so every +# walk re-issues N reflection round-trips proportional to the chain +# length (1 RTT per level past .jvm). X1-1 caches the bound method +# once before its inner loop, hiding this cost — XA exposes it. +# +# A future attribute-memoization PR (issue #557 cold-start work) +# should collapse repeat walks to 0 RTTs after the first; XA is the +# canary for that win. +class _XAAttributeWalkBase(MacroScenario): + """Walk an FQN chain on `gateway.jvm` repeatedly without caching.""" + chain = ("java", "lang", "System") + iterations_per_round = 1_000 + + def measure(self, gateway): + jvm = gateway.jvm + a, b, c = self.chain + for _ in range(self.iterations_per_round): + # Resolve the full chain; no terminal call, so we measure + # only the attribute-resolution path (no actual JVM method + # invocation). + getattr(getattr(getattr(jvm, a), b), c) + + +class XA_AttributeWalk3(_XAAttributeWalkBase): + id = "XA-3" + name = "attribute_walk_jvm_java_lang_System" + # Three-level walk: jvm.java.lang.System -> 3 reflection RTTs per + # iteration on master; should collapse to ~0 after caching lands. + chain = ("java", "lang", "System") + iterations_per_round = 1_000 + + +# ===================================================================== XB +# Gateway reconnect: open a fresh JavaGateway against the *already- +# running* JVM, make one call, close. Measures the per-connection +# setup cost on both sides — Java accept() loop allocates 14 command +# class instances per connection (GatewayConnection.java:196-208), +# Python side runs _create_connection + socket setup + optional auth. +# This is the lever for the Java-side command-prototype-cache PR. +class XB_GatewayReconnect(MacroScenario): + id = "XB" + name = "gateway_reconnect_against_running_jvm" + # One round = 50 reconnect cycles. Empirically ~10-20 ms per cycle + # on M-series + JDK 21, so each timed round is ~0.5-1.0 s — well + # past scheduler jitter. + iterations_per_round = 50 + + def measure(self, gateway): + # Use the running JVM's port. We're a separate client; the + # fixture's gateway stays connected throughout. + from py4j.java_gateway import GatewayParameters + port = gateway.gateway_parameters.port + for _ in range(self.iterations_per_round): + gw = JavaGateway( + gateway_parameters=GatewayParameters(port=port)) + try: + gw.jvm.java.lang.System.currentTimeMillis() + finally: + gw.close() + + +# ===================================================================== XC +# Callback infrastructure overhead: same workload as X1-1 (10k +# currentTimeMillis() calls, single thread) but with the CallbackServer +# started. NO callbacks are actually invoked — the delta vs X1-1 is +# the steady-state cost of running the callback server alongside the +# main gateway. Target of the lazy-CallbackClient PR. +class XC_CallbackInfraOverheadUnused(MacroScenario): + id = "XC" + name = "callback_infra_overhead_unused" + enable_callbacks = True + iterations_per_round = 10_000 + + def measure(self, gateway): + fn = gateway.jvm.java.lang.System.currentTimeMillis + for _ in range(self.iterations_per_round): + fn() + + +# ===================================================================== XD +# Full cold start: spawn a fresh JVM subprocess, build a JavaGateway, +# make one call, shut everything down. One measure() = one cold start. +# Unlike the other scenarios, XD owns its JVM lifecycle inside +# measure() and ignores the fixture-provided gateway (which exists +# only to confirm the classpath builds and skip the test cleanly if +# Java isn't available). Slow per-iteration (~300 ms on M-series + +# JDK 21, ~1-3 s on slower hosts), so CodSpeed will only do a handful +# of rounds. Targets of the AppCDS / TieredStopAtLevel=1 / CRaC work. +class XD_FullColdStart(MacroScenario): + id = "XD" + name = "full_cold_start_subprocess_first_call" + iterations_per_round = 1 + + def measure(self, gateway): + # Import inside measure() so the cost of import isn't paid + # repeatedly; jvm helpers are already imported by the fixture + # in any sane run. + from py4j.tests.perf.jvm import spawn_jvm, shutdown_jvm + process = spawn_jvm() + try: + # Brief sleep mirrors fresh_jvm's startup_sleep so we + # exercise the same path; without it on a fast machine the + # first connect attempt can race ahead of the listening + # socket. + import time + time.sleep(0.25) + gw = None + try: + gw = JavaGateway() + gw.jvm.java.lang.System.currentTimeMillis() + finally: + if gw is not None: + try: + gw.shutdown() + except Exception: + pass + finally: + shutdown_jvm(process, None) + + ALL_MACRO_CLASSES = [ X1_1Thread, X1_4Thread, X1_16Thread, X2_1k, X2_10k, X2_100k, @@ -389,4 +513,13 @@ class X8_256k(_X8BytesSendBase): X6_PoolSaturation, X7_1k, X7_16k, X7_256k, X8_1k, X8_16k, X8_256k, + XA_AttributeWalk3, + XB_GatewayReconnect, + XC_CallbackInfraOverheadUnused, + # XD_FullColdStart is intentionally omitted: it spawns its own + # JVM subprocess inside measure(), so it cannot share the + # fresh_jvm()-managed gateway the framework runner provides + # (port-bind collision on 25333). XD is registered only on the + # CodSpeed path via codspeed_macros._COLD_START_SCENARIOS, where + # it runs against test_cold_start_scenario (no shared fixture). ]