perf: add 4 CodSpeed scenarios for cold-start work (issue #557)

py4j-python/src/py4j/tests/perf/README.md

@@ -142,6 +142,12 @@ Aggregate throughput. Scenario IDs with variants reveal scaling curves.
 | X4 | `Collections.sort` on Python-`Comparable` proxies | Callback round-trip |
 | X5 | 1 000 × `ArrayList.get(-1)` → `Py4JJavaError` | Error-path latency |
 | X6 | 50 concurrent threads, 20 calls each | Pool saturation tail latency |
+| X7-1k / X7-16k / X7-256k | `ByteBuffer.array()` returning N bytes | Bytes recv (decode_bytearray) |
+| X8-1k / X8-16k / X8-256k | `BAOS.write(bytes, 0, N)` of N bytes | Bytes send (encode_bytearray) |
+| XA-3 | 1 000 walks of `gateway.jvm.java.lang.System` | Attribute-resolution cache canary (issue #557) |
+| XB | 50 fresh `JavaGateway()` against a running JVM | Per-connection setup cost (issue #557) |
+| XC | X1-1 workload with CallbackServer started, no callbacks invoked | Callback-infra overhead delta (issue #557) |
+| XD | `subprocess.Popen` → first call → shutdown, one cycle per round | Full cold start (issue #557 — CodSpeed only) |
 
 ## How to read the report
 

py4j-python/src/py4j/tests/perf/scenarios/codspeed_macros.py

@@ -11,8 +11,13 @@
 on the dashboard, so regressions show up per-scenario rather than as one
 opaque aggregate.
 
-Scenario coverage rationale: four macros covering the perf
-characteristics most prone to regression on the py4j socket / call path:
+Scenario coverage rationale: macros covering the perf characteristics
+most prone to regression on the py4j socket / call path. Two test
+functions live here:
+
+``test_macro_scenario`` — scenarios that share a long-running JVM
+provided by the ``macro_gateway`` fixture. Each scenario reuses the
+same gateway across rounds (setup once, measure many).
 
 * ``X1-1``  — single-thread concurrent_1_thread (10k sequential calls).
               Baseline round-trip latency floor.
@@ -23,6 +28,18 @@
               the most complex code path in py4j.
 * ``X6``    — pool_saturation_50_threads. Connection pool behavior
               under high concurrency, tail-latency sensitive.
+* ``X7-16k`` / ``X8-16k`` — bytes recv/send 16k.
+* ``XA-3``  — attribute walk depth (Python __getattr__ cache canary).
+* ``XB``    — gateway reconnect against running JVM (per-connection
+              setup cost).
+* ``XC``    — callback infra overhead unused (delta vs X1-1).
+
+``test_cold_start_scenario`` — scenarios that own their full JVM
+lifecycle inside ``measure()``. Cannot share the fixture's JVM
+because port 25333 is already in use; each round must spawn and
+shut down its own subprocess.
+
+* ``XD``    — full_cold_start_subprocess_first_call.
 
 Adding more scenarios later is one parametrize entry per scenario.
 """
@@ -41,6 +58,10 @@
     X6_PoolSaturation,
     X7_16k,
     X8_16k,
+    XA_AttributeWalk3,
+    XB_GatewayReconnect,
+    XC_CallbackInfraOverheadUnused,
+    XD_FullColdStart,
 )
 
 
@@ -51,9 +72,29 @@
 # invisible to the per-PR dashboard — every prior macro returns
 # int / list / void / callback and the byte path was a measurement
 # blind spot.
+#
+# XA-XC add cold-start / attribute-walk / connection-setup / callback-
+# infra measurement surfaces that share the long-running fixture JVM.
+# Each maps 1:1 to a planned cold-start improvement PR (issue #557):
+#   XA — attribute caches (Python __getattr__ memoization)
+#   XB — Java per-connection command-prototype cache + background warm
+#   XC — lazy CallbackClient (delta vs X1-1 = current overhead)
+# XD lives in _COLD_START_SCENARIOS below — owns its JVM lifecycle.
+# Without these, a per-PR CodSpeed dashboard would not see the wins.
 _MACRO_SCENARIOS = [
     X1_1Thread, X2_10k, X4_Callbacks, X6_PoolSaturation,
     X7_16k, X8_16k,
+    XA_AttributeWalk3, XB_GatewayReconnect,
+    XC_CallbackInfraOverheadUnused,
+]
+
+# Scenarios whose measure() spawns its own JVM subprocess per round.
+# They must NOT share the fixture's JVM — the fixture's JVM is already
+# bound to the default port 25333, so a second spawn would race on
+# bind. Each cold-start scenario is self-contained: spawn, connect,
+# call, shut down.
+_COLD_START_SCENARIOS = [
+    XD_FullColdStart,
 ]
 
 
@@ -94,3 +135,30 @@ def test_macro_scenario(benchmark, macro_gateway):
     if hasattr(scenario, "setup"):
         scenario.setup(gateway)
     benchmark(scenario.measure, gateway)
+
+
+@pytest.mark.parametrize(
+    "scenario_cls", _COLD_START_SCENARIOS,
+    ids=[cls.id for cls in _COLD_START_SCENARIOS],
+)
+def test_cold_start_scenario(benchmark, scenario_cls):
+    """Run a cold-start scenario that owns its JVM lifecycle.
+
+    No shared fixture JVM — each ``measure()`` invocation spawns a
+    fresh subprocess, runs one full cold-start cycle, and tears it
+    down. Slow per-iteration (~300 ms on M-series + JDK 21, multiple
+    seconds on slower hosts); CodSpeed adapts iteration count
+    accordingly.
+
+    Skips cleanly if the Java side hasn't been built — checked via the
+    same ``verify_classpath`` path used by ``fresh_jvm``.
+    """
+    from py4j.tests.perf.jvm import verify_classpath
+    try:
+        verify_classpath()
+    except JvmNotBuiltError as e:
+        pytest.skip(str(e))
+    scenario = scenario_cls()
+    # Cold-start scenarios have no shared state; setup() is not
+    # expected. measure() ignores the gateway arg (it owns its own).
+    benchmark(scenario.measure, None)

py4j-python/src/py4j/tests/perf/scenarios/macro.py

@@ -9,6 +9,7 @@
 
 import threading
 
+from py4j.java_gateway import JavaGateway
 from py4j.tests.perf.runner import MacroScenario
 
 
@@ -380,6 +381,129 @@ class X8_256k(_X8BytesSendBase):
     iterations_per_round = 25
 
 
+# ===================================================================== XA
+# Attribute walk depth: re-walks the gateway.jvm.java.lang.System
+# attribute chain many times. py4j's JVMView / JavaPackage / JavaClass
+# __getattr__ methods do NOT memoize their results today, so every
+# walk re-issues N reflection round-trips proportional to the chain
+# length (1 RTT per level past .jvm). X1-1 caches the bound method
+# once before its inner loop, hiding this cost — XA exposes it.
+#
+# A future attribute-memoization PR (issue #557 cold-start work)
+# should collapse repeat walks to 0 RTTs after the first; XA is the
+# canary for that win.
+class _XAAttributeWalkBase(MacroScenario):
+    """Walk an FQN chain on `gateway.jvm` repeatedly without caching."""
+    chain = ("java", "lang", "System")
+    iterations_per_round = 1_000
+
+    def measure(self, gateway):
+        jvm = gateway.jvm
+        a, b, c = self.chain
+        for _ in range(self.iterations_per_round):
+            # Resolve the full chain; no terminal call, so we measure
+            # only the attribute-resolution path (no actual JVM method
+            # invocation).
+            getattr(getattr(getattr(jvm, a), b), c)
+
+
+class XA_AttributeWalk3(_XAAttributeWalkBase):
+    id = "XA-3"
+    name = "attribute_walk_jvm_java_lang_System"
+    # Three-level walk: jvm.java.lang.System -> 3 reflection RTTs per
+    # iteration on master; should collapse to ~0 after caching lands.
+    chain = ("java", "lang", "System")
+    iterations_per_round = 1_000
+
+
+# ===================================================================== XB
+# Gateway reconnect: open a fresh JavaGateway against the *already-
+# running* JVM, make one call, close. Measures the per-connection
+# setup cost on both sides — Java accept() loop allocates 14 command
+# class instances per connection (GatewayConnection.java:196-208),
+# Python side runs _create_connection + socket setup + optional auth.
+# This is the lever for the Java-side command-prototype-cache PR.
+class XB_GatewayReconnect(MacroScenario):
+    id = "XB"
+    name = "gateway_reconnect_against_running_jvm"
+    # One round = 50 reconnect cycles. Empirically ~10-20 ms per cycle
+    # on M-series + JDK 21, so each timed round is ~0.5-1.0 s — well
+    # past scheduler jitter.
+    iterations_per_round = 50
+
+    def measure(self, gateway):
+        # Use the running JVM's port. We're a separate client; the
+        # fixture's gateway stays connected throughout.
+        from py4j.java_gateway import GatewayParameters
+        port = gateway.gateway_parameters.port
+        for _ in range(self.iterations_per_round):
+            gw = JavaGateway(
+                gateway_parameters=GatewayParameters(port=port))
+            try:
+                gw.jvm.java.lang.System.currentTimeMillis()
+            finally:
+                gw.close()
+
+
+# ===================================================================== XC
+# Callback infrastructure overhead: same workload as X1-1 (10k
+# currentTimeMillis() calls, single thread) but with the CallbackServer
+# started. NO callbacks are actually invoked — the delta vs X1-1 is
+# the steady-state cost of running the callback server alongside the
+# main gateway. Target of the lazy-CallbackClient PR.
+class XC_CallbackInfraOverheadUnused(MacroScenario):
+    id = "XC"
+    name = "callback_infra_overhead_unused"
+    enable_callbacks = True
+    iterations_per_round = 10_000
+
+    def measure(self, gateway):
+        fn = gateway.jvm.java.lang.System.currentTimeMillis
+        for _ in range(self.iterations_per_round):
+            fn()
+
+
+# ===================================================================== XD
+# Full cold start: spawn a fresh JVM subprocess, build a JavaGateway,
+# make one call, shut everything down. One measure() = one cold start.
+# Unlike the other scenarios, XD owns its JVM lifecycle inside
+# measure() and ignores the fixture-provided gateway (which exists
+# only to confirm the classpath builds and skip the test cleanly if
+# Java isn't available). Slow per-iteration (~300 ms on M-series +
+# JDK 21, ~1-3 s on slower hosts), so CodSpeed will only do a handful
+# of rounds. Targets of the AppCDS / TieredStopAtLevel=1 / CRaC work.
+class XD_FullColdStart(MacroScenario):
+    id = "XD"
+    name = "full_cold_start_subprocess_first_call"
+    iterations_per_round = 1
+
+    def measure(self, gateway):
+        # Import inside measure() so the cost of import isn't paid
+        # repeatedly; jvm helpers are already imported by the fixture
+        # in any sane run.
+        from py4j.tests.perf.jvm import spawn_jvm, shutdown_jvm
+        process = spawn_jvm()
+        try:
+            # Brief sleep mirrors fresh_jvm's startup_sleep so we
+            # exercise the same path; without it on a fast machine the
+            # first connect attempt can race ahead of the listening
+            # socket.
+            import time
+            time.sleep(0.25)
+            gw = None
+            try:
+                gw = JavaGateway()
+                gw.jvm.java.lang.System.currentTimeMillis()
+            finally:
+                if gw is not None:
+                    try:
+                        gw.shutdown()
+                    except Exception:
+                        pass
+        finally:
+            shutdown_jvm(process, None)
+
+
 ALL_MACRO_CLASSES = [
     X1_1Thread, X1_4Thread, X1_16Thread,
     X2_1k, X2_10k, X2_100k,
@@ -389,4 +513,13 @@ class X8_256k(_X8BytesSendBase):
     X6_PoolSaturation,
     X7_1k, X7_16k, X7_256k,
     X8_1k, X8_16k, X8_256k,
+    XA_AttributeWalk3,
+    XB_GatewayReconnect,
+    XC_CallbackInfraOverheadUnused,
+    # XD_FullColdStart is intentionally omitted: it spawns its own
+    # JVM subprocess inside measure(), so it cannot share the
+    # fresh_jvm()-managed gateway the framework runner provides
+    # (port-bind collision on 25333). XD is registered only on the
+    # CodSpeed path via codspeed_macros._COLD_START_SCENARIOS, where
+    # it runs against test_cold_start_scenario (no shared fixture).
 ]