From fcd60b164090fab3d1659d6c4492f03fb29b9810 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 11:18:08 +0200 Subject: [PATCH 1/3] refactor(provider-tck): extract BackendControl and split the base class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step definitions reached the Compose stack and the HTTP control API directly, through TckRuntime. That made the suite unrunnable for any provider without a containerised backend, and it put transport knowledge in the one layer that should have none. Introduce BackendControl as the single seam between the step definitions and whatever manipulates the backend. All nine touchpoints — scenario reset, flag change, disconnect, reconnect, bounded outage, provider creation and the suite lifecycle — now go through it. ControlApiClient becomes HttpBackendControl, one implementation of that seam; nothing about the HTTP control API spec changes and it remains the normative contract for external backends. Split the base class along the same line. ProviderTckTest (renamed from AbstractProviderTckTest) keeps only what every provider needs: capability declaration, timeouts, awaiting and step wiring. ContainerizedProviderTckTest extends it with the Compose lifecycle, port discovery and HttpBackendControl construction, and carries the compose-specific configuration that used to sit on ProviderTckHarness. Adopters with an external backend keep an unchanged surface — the flagd suites need only the superclass name. Behaviour is unchanged: same control API calls in the same order, same once-per-suite Compose lifecycle, same no-container-restart invariant. Signed-off-by: Simon Schrottner --- .../flagd/e2e/AbstractFlagdTckTest.java | 4 +- tools/provider-tck/pom.xml | 16 +- .../tools/providertck/BackendControl.java | 118 ++++++++ .../tools/providertck/BackendEndpoint.java | 6 +- .../ContainerizedProviderTckTest.java | 272 ++++++++++++++++++ ...ApiClient.java => HttpBackendControl.java} | 192 +++++++------ .../tools/providertck/ProviderTckHarness.java | 187 +++++------- ...viderTckTest.java => ProviderTckTest.java} | 35 ++- .../contrib/tools/providertck/TckRuntime.java | 125 ++++---- .../providertck/steps/AbstractSteps.java | 26 +- .../providertck/steps/ProviderSteps.java | 45 +-- 11 files changed, 703 insertions(+), 323 deletions(-) create mode 100644 tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java create mode 100644 tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java rename tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/{ControlApiClient.java => HttpBackendControl.java} (63%) rename tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/{AbstractProviderTckTest.java => ProviderTckTest.java} (52%) diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java index 8c3f833e2..4b712a078 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java @@ -3,9 +3,9 @@ import dev.openfeature.contrib.providers.flagd.Config; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.FlagdProvider; -import dev.openfeature.contrib.tools.providertck.AbstractProviderTckTest; import dev.openfeature.contrib.tools.providertck.BackendEndpoint; import dev.openfeature.contrib.tools.providertck.Capability; +import dev.openfeature.contrib.tools.providertck.ContainerizedProviderTckTest; import dev.openfeature.sdk.FeatureProvider; import java.io.File; import java.util.Collections; @@ -24,7 +24,7 @@ * one is running from the JUnit test plan, so adding a mode needs no registration or build * configuration. */ -abstract class AbstractFlagdTckTest extends AbstractProviderTckTest { +abstract class AbstractFlagdTckTest extends ContainerizedProviderTckTest { /** * A port nothing listens on, for the initialisation-failure scenarios. diff --git a/tools/provider-tck/pom.xml b/tools/provider-tck/pom.xml index 3de834808..a92c7c715 100644 --- a/tools/provider-tck/pom.xml +++ b/tools/provider-tck/pom.xml @@ -25,11 +25,13 @@ provider-tck Language-agnostic conformance test suite (TCK) for OpenFeature providers. - Bundles the canonical Gherkin feature files, Cucumber step definitions and an - abstract JUnit Platform Suite base class that owns the full test lifecycle: - starting the vendor-supplied Docker Compose stack, discovering dynamically - mapped ports, driving the standardised control API and awaiting provider - events. Provider authors implement a single factory interface. + Bundles the canonical Gherkin feature files, Cucumber step definitions and + abstract JUnit Platform Suite base classes that own the full test lifecycle. + Providers with an external backend extend ContainerizedProviderTckTest, which + starts the vendor-supplied Docker Compose stack, discovers dynamically mapped + ports and drives the standardised HTTP control API. Providers with no backend + extend ProviderTckTest and supply in-process backend control. Provider authors + implement a small factory interface either way. https://openfeature.dev @@ -97,7 +99,7 @@ + OBJECT_FACTORY_PROPERTY_NAME constants used in ProviderTckTest --> io.cucumber cucumber-junit-platform-engine @@ -110,7 +112,7 @@ compile - org.junit.platform diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java new file mode 100644 index 000000000..820ffdaa8 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java @@ -0,0 +1,118 @@ +package dev.openfeature.contrib.tools.providertck; + +import java.time.Duration; + +/** + * The single seam between the TCK's step definitions and whatever manipulates the backend. + * + *

Step definitions never talk to a backend directly. They talk to this interface, which is why + * the same Gherkin can run unchanged against any backend an implementation can drive — + * a containerised one over HTTP ({@link HttpBackendControl}) being the first. Nothing below this + * line knows about ports, containers or transports. + * + *

Which implementation is right for your provider

+ * + *

If your provider talks to a backend — a server, a service, anything out of process — use + * {@link HttpBackendControl} by extending {@link ContainerizedProviderTckTest}. The HTTP control + * API in {@code openapi/control-api.yaml} is the normative contract for those providers, and it is + * what makes a conformance claim portable: another language's TCK drives the same endpoints against + * the same stack and must get the same answers. + * + *

Do not write a custom in-JVM {@code BackendControl} that reaches into an + * external backend through a side channel — a test-only admin client, a shared database handle, a + * static hook inside the provider. It will pass, and it will prove nothing, because the thing it + * exercised is not the thing the contract describes. + * + *

An in-JVM implementation is legitimate only for providers that have no backend to + * contract with: in-memory, environment-variable and file-based providers, where "the backend" is a + * data structure in the same JVM. + * + *

Operations a backend may not support

+ * + *

{@link #prepareScenario()} and {@link #changeFlag()} are mandatory: a backend that cannot reset + * itself or change a flag cannot run the suite at all. + * + *

The three connection operations are not. A provider with nothing to disconnect from leaves + * them at their defaults, which throw {@link UnsupportedOperationException}. That exception is a + * test-configuration bug, never a skip — the scenarios that need connection + * control are gated behind {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT}, so + * reaching one of these defaults means a capability was declared that the backend cannot back up. + * Failing loudly there is deliberate: a silent no-op would report the scenario as passed. + * + * @see Capability + * @see ProviderTckTest + */ +public interface BackendControl { + + /** + * Brings the backend to the state every scenario starts from: reachable, with flag state at the + * baseline of the canonical flag set. + * + *

Called once before each scenario. This is the TCK's only isolation mechanism — scenarios + * share one backend for the whole suite, and containers are never restarted between them. + */ + void prepareScenario(); + + /** + * Mutates flag configuration so that a conforming provider observes a configuration change and + * resolves a different value for {@code changing-flag} afterwards. + * + *

Which value it changes to is deliberately unspecified; the suite asserts only that the + * resolved value differs from what it was before. + */ + void changeFlag(); + + /** + * Makes the backend unreachable for the rest of the scenario, without stopping any container. + * + * @throws UnsupportedOperationException if this backend has no connection to lose + */ + default void disconnect() { + throw unsupported("disconnect"); + } + + /** + * Makes the backend reachable again after {@link #disconnect()}, preserving flag state so the + * provider observes an availability change rather than a configuration change. + * + * @throws UnsupportedOperationException if this backend has no connection to restore + */ + default void reconnect() { + throw unsupported("reconnect"); + } + + /** + * Makes the backend unreachable for a bounded period, after which it comes back on its own. + * + * @param outage how long the backend stays unreachable + * @throws UnsupportedOperationException if this backend has no connection to lose + */ + default void disconnectFor(Duration outage) { + throw unsupported("disconnectFor"); + } + + /** + * Returns a short description of what is being controlled, for startup logging and for the + * failure messages of unsupported operations. + * + * @return a human-readable description of this backend control + */ + default String description() { + return getClass().getSimpleName(); + } + + /** + * Builds the exception the connection-control defaults throw. + * + * @param operation the operation that is not supported + * @return the exception to throw + */ + default UnsupportedOperationException unsupported(String operation) { + return new UnsupportedOperationException(description() + " does not support '" + operation + + "'. This is a test-configuration bug rather than a provider defect: a scenario " + + "needing connection control ran, so the harness declared Capability.STALE or " + + "Capability.UNAVAILABLE_INIT for a backend that cannot simulate an outage. " + + "Remove those capabilities from the harness, or supply a BackendControl that " + + "implements them."); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java index 40f7799cf..b3d89d587 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java @@ -4,7 +4,7 @@ /** * Addresses of the running backend stack, handed to - * {@link ProviderTckHarness#createProvider(BackendEndpoint)}. + * {@link ContainerizedProviderTckTest#createProvider(BackendEndpoint)}. * *

This type exists because external ports are only known after the Compose stack has * started. Compose stacks under test must not pin host ports — Docker assigns them dynamically, so @@ -53,7 +53,7 @@ public String host(String service) { * backend service. * * @param internalPort the container-internal port, as declared by - * {@link ProviderTckHarness#backendPorts()} + * {@link ContainerizedProviderTckTest#backendPorts()} * @return the host port the service is reachable on */ public int port(int internalPort) { @@ -64,7 +64,7 @@ public int port(int internalPort) { * Resolves the dynamically mapped host port for a container-internal port on a named service. * *

Use this for multi-service stacks — a proxy, an edge service, a sidecar. The service and - * port must have been declared via {@link ProviderTckHarness#additionalExposedPorts()}, + * port must have been declared via {@link ContainerizedProviderTckTest#additionalExposedPorts()}, * otherwise Testcontainers has not exposed it and this call fails. * * @param service the Compose service name diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java new file mode 100644 index 000000000..63bfefe13 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java @@ -0,0 +1,272 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.FeatureProvider; +import java.io.File; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +/** + * Base JUnit Platform Suite for providers that talk to an external backend. + * + *

Adds everything {@link ProviderTckTest} deliberately leaves out: the Docker Compose lifecycle, + * discovery of dynamically mapped host ports, and construction of an {@link HttpBackendControl} + * against the backend's control API. This is the base class for the overwhelming majority of + * providers. + * + *

The HTTP control API described in {@code openapi/control-api.yaml} is the normative contract + * here, and that is the point: another language's TCK drives the same endpoints against the same + * stack and must get the same answers. Substituting a custom in-JVM {@link BackendControl} that + * manipulates an external backend through a side channel bypasses that contract — see + * {@link BackendControl} for why that is not an acceptable adoption path. + * + *

Provider authors implement three methods, optionally a fourth, and override the defaults their + * stack needs. The Compose stack is started once, before the first scenario, and + * stopped after the last one. It is never stopped or restarted in between: Testcontainers cannot + * reliably preserve dynamically mapped host ports across a container restart, so a restart would + * silently invalidate every provider already pointed at the old port. Backend unavailability is + * therefore always simulated inside the running stack through the control API. + * + *

Example — the entire adoption for a provider with one transport: + * + * 

{@code
+ * public class MyProviderTckTest extends ContainerizedProviderTckTest {
+ *
+ *     @Override
+ *     public File composeFile() {
+ *         return new File("src/test/resources/tck/docker-compose.yaml");
+ *     }
+ *
+ *     @Override
+ *     public List backendPorts() {
+ *         return Collections.singletonList(8013);
+ *     }
+ *
+ *     @Override
+ *     public FeatureProvider createProvider(BackendEndpoint endpoint) {
+ *         return new MyProvider(endpoint.host(), endpoint.port(8013));
+ *     }
+ *
+ *     @Override
+ *     public FeatureProvider createUnavailableProvider() {
+ *         return new MyProvider("localhost", 9999);
+ *     }
+ * }
+ * }
+ * + * @see ProviderTckTest + * @see HttpBackendControl + */ +public abstract class ContainerizedProviderTckTest extends ProviderTckTest { + + private static final Logger log = LoggerFactory.getLogger(ContainerizedProviderTckTest.class); + + private ComposeContainer compose; + private BackendEndpoint endpoint; + private HttpBackendControl control; + + // --------------------------------------------------------------------------------------- + // What a provider author supplies + // --------------------------------------------------------------------------------------- + + /** + * Returns the Docker Compose file describing the backend stack under test. + * + *

The path is resolved relative to the Maven module directory, so + * {@code new File("src/test/resources/tck/docker-compose.yaml")} is the idiomatic form. + * + *

The stack must not pin host ports — Docker assigns them dynamically and the TCK discovers + * them after startup. + * + * @return the Compose file describing the backend stack + */ + public abstract File composeFile(); + + /** + * Returns the container-internal ports on {@link #backendService()} that the provider connects + * to, so Testcontainers can expose and map them. + * + *

The control API port from {@link #controlPort()} is exposed automatically and does not + * need to be listed here. + * + * @return container-internal ports the provider connects to + */ + public abstract List backendPorts(); + + /** + * Creates the provider under test, configured against the running backend. + * + *

Called after the Compose stack is up and the control API has seeded the canonical flag + * set. The endpoint carries the dynamically mapped host ports, which is why this is a factory + * rather than a field: the ports do not exist until the stack has started. + * + *

The TCK owns the provider lifecycle from here. Do not call {@code setProvider} or + * {@code initialize} yourself. + * + * @param endpoint host and mapped ports of the running backend stack + * @return a configured, uninitialised provider + */ + public abstract FeatureProvider createProvider(BackendEndpoint endpoint); + + /** + * {@inheritDoc} + * + *

Delegates to {@link #createProvider(BackendEndpoint)} with the running stack's endpoint. + */ + @Override + public final FeatureProvider createProvider() { + return createProvider(endpoint()); + } + + /** + * {@inheritDoc} + * + *

Abstract here rather than defaulted: a provider with a real backend can always be pointed + * at a closed port, so there is no reason for one not to cover the initialisation-failure + * scenarios. + */ + @Override + public abstract FeatureProvider createUnavailableProvider(); + + /** + * Returns the Compose service name that hosts the control API and the backend the provider + * connects to. + * + * @return the Compose service name, {@code backend} by default + */ + public String backendService() { + return "backend"; + } + + /** + * Returns the container-internal port the control API listens on. + * + * @return the control API port, {@code 8080} by default + */ + public int controlPort() { + return 8080; + } + + /** + * Returns extra services and container-internal ports to expose, for stacks that contain more + * than the backend service. + * + *

Keys are Compose service names, values are container-internal ports. Resolve the mapped + * ports with {@link BackendEndpoint#port(String, int)}. + * + * @return additional services and ports to expose, empty by default + */ + public Map> additionalExposedPorts() { + return Collections.emptyMap(); + } + + /** + * Returns the control API configuration name used to seed the canonical flag set. + * + * @return the configuration name passed to {@code POST /start}, {@code default} by default + */ + public String defaultConfig() { + return "default"; + } + + /** + * Returns how long to wait for the Compose stack and its control API to become reachable. + * + * @return the stack startup timeout, 60 seconds by default + */ + public Duration startupTimeout() { + return Duration.ofSeconds(60); + } + + /** + * Returns how long to pause after a control API call before continuing. + * + *

Covers the gap between the control API acknowledging a command and the backend actually + * having acted on it. Raise it if you see flakiness immediately after + * {@code the flag was modified} or a provider setup step. + * + * @return the settle time, 50 milliseconds by default + */ + public Duration settleTime() { + return Duration.ofMillis(50); + } + + // --------------------------------------------------------------------------------------- + // The lifecycle-agnostic contract, implemented in terms of the Compose stack + // --------------------------------------------------------------------------------------- + + /** + * {@inheritDoc} + * + *

Starts the Compose stack, resolves the control API's mapped port and waits for it to + * accept commands. + */ + @Override + public final void startSuite() { + compose = startCompose(); + endpoint = new BackendEndpoint(compose, backendService()); + control = new HttpBackendControl( + "http://" + endpoint.host() + ":" + endpoint.port(controlPort()), defaultConfig(), settleTime()); + control.awaitReady(startupTimeout()); + log.info("Control API ready at {}", control.baseUrl()); + } + + /** {@inheritDoc} */ + @Override + public final void stopSuite() { + if (compose != null) { + compose.stop(); + } + compose = null; + endpoint = null; + control = null; + } + + /** {@inheritDoc} */ + @Override + public final BackendControl backendControl() { + return control; + } + + /** + * Returns the host and mapped ports of the running stack. + * + * @return the backend endpoint + * @throws IllegalStateException if the stack has not been started + */ + protected final BackendEndpoint endpoint() { + if (endpoint == null) { + throw new IllegalStateException("The Compose stack has not been started yet."); + } + return endpoint; + } + + private ComposeContainer startCompose() { + File composeFile = composeFile(); + if (!composeFile.isFile()) { + throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() + + ". ContainerizedProviderTckTest.composeFile() is resolved relative to the module directory."); + } + ComposeContainer stack = new ComposeContainer(composeFile); + + stack.withExposedService(backendService(), controlPort(), Wait.forListeningPort()); + for (Integer port : backendPorts()) { + stack.withExposedService(backendService(), port, Wait.forListeningPort()); + } + for (Map.Entry> service : additionalExposedPorts().entrySet()) { + for (Integer port : service.getValue()) { + stack.withExposedService(service.getKey(), port, Wait.forListeningPort()); + } + } + stack.withStartupTimeout(startupTimeout()); + + log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); + stack.start(); + return stack; + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/HttpBackendControl.java similarity index 63% rename from tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java rename to tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/HttpBackendControl.java index 6c0a19e57..42183b52c 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/HttpBackendControl.java @@ -10,38 +10,52 @@ import org.slf4j.LoggerFactory; /** - * Client for the standardised backend control API. + * {@link BackendControl} backed by the standardised HTTP control API. * - *

Implements the contract in {@code openapi/control-api.yaml}, including the documented fallback - * for the optional {@code /reset} operation. Uses the JDK HTTP client so that adopting the TCK does - * not drag an HTTP library onto a provider's test classpath. + *

This is the normative implementation for every provider that talks to an external backend. It + * implements the contract in {@code openapi/control-api.yaml}, including the documented fallback + * for the optional {@code /reset} operation. It uses the JDK HTTP client so that adopting the TCK + * does not drag an HTTP library onto a provider's test classpath. * *

Every operation here manipulates the backend process or its flag state. None of them * touch containers — that is the no-container-restart invariant, and it is the reason a provider * built once at suite start stays valid for every scenario. + * + *

Constructed by {@link ContainerizedProviderTckTest} once the Compose stack is up and the + * control API host port is known. Provider authors do not build one themselves. */ -public final class ControlApiClient { +public final class HttpBackendControl implements BackendControl { - private static final Logger log = LoggerFactory.getLogger(ControlApiClient.class); + private static final Logger log = LoggerFactory.getLogger(HttpBackendControl.class); private final HttpClient http; private final String baseUrl; + private final String defaultConfig; private final Duration settleTime; /** * Tri-state cache of whether the backend implements the optional {@code /reset} operation. - * {@code null} until the first {@link #reset(String)} call probes it. + * {@code null} until the first {@link #reset()} call probes it. */ private Boolean resetSupported; /** - * Whether the backend was last known to be unreachable. Conservative: {@code restart} sets it - * even though the backend comes back on its own, because a scenario may end before it does. + * Whether the backend was last known to be unreachable. Conservative: {@link #disconnectFor} + * sets it even though the backend comes back on its own, because a scenario may end before it + * does. */ private boolean backendStopped; - ControlApiClient(String baseUrl, Duration settleTime) { + /** + * Creates a control client for a running backend. + * + * @param baseUrl the control API base URL, without a trailing slash + * @param defaultConfig the configuration name defining the canonical baseline + * @param settleTime how long to pause after a command before continuing + */ + HttpBackendControl(String baseUrl, String defaultConfig, Duration settleTime) { this.baseUrl = baseUrl; + this.defaultConfig = defaultConfig; this.settleTime = settleTime; this.http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); @@ -56,105 +70,80 @@ public String baseUrl() { return baseUrl; } - /** - * Starts the backend with a named configuration, seeding flag state to that configuration's - * baseline. - * - * @param config the configuration name - */ - public void start(String config) { - post("/start?config=" + config); - backendStopped = false; + @Override + public String description() { + return "HTTP control API at " + baseUrl; } /** - * Makes the backend unreachable without stopping its container. + * {@inheritDoc} + * + *

Prefers {@code POST /reset} when the backend is already running, because restoring the + * baseline without an availability blip means the previous scenario teardown cannot leak a + * spurious lifecycle event into the next scenario. When the previous scenario left the backend + * unreachable, {@code /reset} alone would not bring it back, so this falls through to + * {@code POST /start?config=...}. */ - public void stop() { - post("/stop"); - backendStopped = true; + @Override + public void prepareScenario() { + if (backendStopped) { + start(); + } else { + reset(); + } } /** - * Makes the backend unreachable for a bounded duration, then starts it again. - * - *

Flag state is preserved across the outage, so a provider observes an availability change - * and not a configuration change. + * {@inheritDoc} * - * @param seconds how long the backend stays unreachable + *

Issues {@code POST /change}. */ - public void restart(int seconds) { - post("/restart?seconds=" + seconds); - backendStopped = true; + @Override + public void changeFlag() { + post("/change"); } /** - * Puts the backend into the state every scenario starts from: running, with flag state at the - * baseline of the default configuration. + * {@inheritDoc} * - *

Prefers {@link #reset(String)} when the backend is already running, because restoring the - * baseline without an availability blip means the previous scenario's teardown cannot leak a - * spurious lifecycle event into the next scenario. When the previous scenario left the backend - * unreachable, {@code /reset} alone would not bring it back, so this falls through to - * {@link #start(String)}. - * - * @param defaultConfig the configuration name defining the baseline + *

Issues {@code POST /stop}, which makes the backend unreachable without stopping its + * container. */ - public void prepareScenario(String defaultConfig) { - if (backendStopped) { - start(defaultConfig); - } else { - reset(defaultConfig); - } + @Override + public void disconnect() { + post("/stop"); + backendStopped = true; } /** - * Mutates flag configuration so that a conforming provider observes a configuration change and - * resolves a different value for {@code changing-flag} afterwards. + * {@inheritDoc} + * + *

Issues {@code POST /start?config=...}, which also restores the baseline flag state. */ - public void change() { - post("/change"); + @Override + public void reconnect() { + start(); } /** - * Restores flag state to the seeded baseline for scenario isolation. - * - *

Prefers the optional {@code POST /reset}, which causes no availability blip. When the - * backend answers {@code 404} or {@code 501} the result is cached and every subsequent call - * falls back to {@code POST /start?config=...}, which resets state at the cost of a process - * restart. Both paths are conformant; see {@code openapi/control-api.yaml}. + * {@inheritDoc} * - * @param defaultConfig the configuration name to fall back to + *

Issues {@code POST /restart?seconds=...}. Flag state is preserved across the outage, so a + * provider observes an availability change and not a configuration change. The control API + * takes whole seconds, so a sub-second outage is rounded up to one second. */ - public void reset(String defaultConfig) { - if (Boolean.FALSE.equals(resetSupported)) { - start(defaultConfig); - return; - } - HttpResponse response = send("/reset"); - if (response.statusCode() == 404 || response.statusCode() == 501) { - if (resetSupported == null) { - log.info( - "Control API at {} does not implement POST /reset (HTTP {}); " - + "falling back to POST /start?config={} for scenario isolation.", - baseUrl, - response.statusCode(), - defaultConfig); - } - resetSupported = false; - start(defaultConfig); - return; - } - expectSuccess("/reset", response); - resetSupported = true; - settle(); + @Override + public void disconnectFor(Duration outage) { + int seconds = (int) Math.max(1, Math.ceil(outage.toMillis() / 1000.0)); + post("/restart?seconds=" + seconds); + backendStopped = true; } /** * Waits until the control API accepts commands. * - *

Probes the optional {@code GET /healthz}. A {@code 404} is a conformant answer meaning "not - * implemented", in which case readiness has already been established by the Testcontainers + *

Probes the optional {@code GET /healthz}. A {@code 404} is a conformant answer meaning + * "not implemented", in which case readiness has already been established by the Testcontainers * listening-port wait strategy and this returns immediately. * * @param timeout how long to keep probing @@ -185,6 +174,47 @@ public void awaitReady(Duration timeout) { throw new IllegalStateException("control API at " + baseUrl + " did not become ready within " + timeout, last); } + /** + * Starts the backend with the default configuration, seeding flag state to that configuration + * baseline. + */ + private void start() { + post("/start?config=" + defaultConfig); + backendStopped = false; + } + + /** + * Restores flag state to the seeded baseline without an availability blip. + * + *

{@code POST /reset} is optional. When the backend answers {@code 404} or {@code 501} the + * result is cached and every subsequent call falls back to {@code POST /start?config=...}, + * which resets state at the cost of a process restart. Both paths are conformant; see + * {@code openapi/control-api.yaml}. + */ + private void reset() { + if (Boolean.FALSE.equals(resetSupported)) { + start(); + return; + } + HttpResponse response = send("/reset"); + if (response.statusCode() == 404 || response.statusCode() == 501) { + if (resetSupported == null) { + log.info( + "Control API at {} does not implement POST /reset (HTTP {}); " + + "falling back to POST /start?config={} for scenario isolation.", + baseUrl, + response.statusCode(), + defaultConfig); + } + resetSupported = false; + start(); + return; + } + expectSuccess("/reset", response); + resetSupported = true; + settle(); + } + private void post(String path) { expectSuccess(path, send(path)); settle(); diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java index 1cc671faf..72dd14c10 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java @@ -1,97 +1,86 @@ package dev.openfeature.contrib.tools.providertck; import dev.openfeature.sdk.FeatureProvider; -import java.io.File; import java.time.Duration; -import java.util.Collections; import java.util.EnumSet; -import java.util.List; -import java.util.Map; import java.util.Set; /** - * The complete contract a provider author implements to run the OpenFeature Provider TCK. + * The lifecycle-agnostic contract a provider author implements to run the OpenFeature Provider TCK. * - *

Four methods have no default and must be supplied. Everything else is a convention with a - * working default. If you find yourself needing to add lifecycle code, container handling or event - * plumbing to your implementation, that is a bug in the TCK's base class rather than something to - * work around here. + *

Two methods have no default: what provider to test, and what manipulates the backend it reads + * from. Everything else is a convention with a working default. Nothing here mentions containers, + * ports or HTTP — that belongs to {@link ContainerizedProviderTckTest}, which implements this + * interface in terms of a Compose stack. * - *

Implementations are discovered through {@link java.util.ServiceLoader}. Extend - * {@link AbstractProviderTckTest} — which implements this interface and carries all the Cucumber - * configuration — and register the concrete class in - * {@code META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness}. + *

Which base class to extend: * - *

Example — the entire adoption for a provider: + *

    + *
  • Your provider talks to an external backend — extend {@link ContainerizedProviderTckTest}. + * It brings the Compose lifecycle, port discovery and {@link HttpBackendControl}, and the + * HTTP control API stays the normative contract for your conformance claim. + *
  • Your provider has no backend (in-memory, environment variables, a local file) — extend + * {@link ProviderTckTest} directly and supply an in-process {@link BackendControl}. + *
+ * + *

Implementations are discovered through the executing JUnit suite, and through + * {@link java.util.ServiceLoader} as a fallback. Extend one of the two base classes — each is both + * the JUnit suite and the harness — and no registration is needed. + * + *

Example — the entire adoption for a backend-less provider: * * 

{@code
- * public class MyProviderTckTest extends AbstractProviderTckTest {
+ * public class MyProviderTckTest extends ProviderTckTest {
  *
- *     @Override
- *     public File composeFile() {
- *         return new File("src/test/resources/tck/docker-compose.yaml");
- *     }
+ *     private final MyInProcessControl control = new MyInProcessControl();
  *
  *     @Override
- *     public List backendPorts() {
- *         return Collections.singletonList(8013);
+ *     public BackendControl backendControl() {
+ *         return control;
  *     }
  *
  *     @Override
- *     public FeatureProvider createProvider(BackendEndpoint endpoint) {
- *         return new MyProvider(endpoint.host(), endpoint.port(8013));
+ *     public FeatureProvider createProvider() {
+ *         return control.createProvider();
  *     }
  *
  *     @Override
- *     public FeatureProvider createUnavailableProvider() {
- *         return new MyProvider("localhost", 9999);
+ *     public Set capabilities() {
+ *         return EnumSet.of(Capability.EVENTS, Capability.CONFIGURATION_CHANGE, Capability.OBJECT);
  *     }
  * }
  * }
+ * + * @see ProviderTckTest + * @see ContainerizedProviderTckTest */ public interface ProviderTckHarness { /** - * Returns the Docker Compose file describing the backend stack under test. + * Creates the provider under test, configured against a backend that is already running and + * seeded with the canonical flag set. * - *

The path is resolved relative to the Maven module directory, so - * {@code new File("src/test/resources/tck/docker-compose.yaml")} is the idiomatic form. - * - *

The stack is started once before the first scenario and stopped after the last one. It is - * never restarted in between — see {@link #createProvider(BackendEndpoint)} and the - * no-container-restart invariant documented in {@code openapi/control-api.yaml}. The stack must - * not pin host ports. - * - * @return the Compose file describing the backend stack - */ - File composeFile(); - - /** - * Returns the container-internal ports on {@link #backendService()} that the provider connects - * to, so Testcontainers can expose and map them. + *

Called once per scenario. This is a factory rather than a field because a provider cannot + * always be configured before the suite starts — a Compose stack's host ports do not exist + * until it is up — and because each scenario gets its own provider instance. * - *

The control API port from {@link #controlPort()} is exposed automatically and does not - * need to be listed here. + *

The TCK owns the provider lifecycle from here: it registers the provider with the + * OpenFeature API under a scenario-scoped domain, waits for it to become ready, and shuts it + * down afterwards. Do not call {@code setProvider} or {@code initialize} yourself. * - * @return container-internal ports the provider connects to + * @return a configured, uninitialised provider */ - List backendPorts(); + FeatureProvider createProvider(); /** - * Creates the provider under test, configured against the running backend. + * Returns the seam through which the TCK manipulates the backend. * - *

Called after the Compose stack is up and the control API has seeded the canonical flag - * set. The endpoint carries the dynamically mapped host ports, which is why this is a factory - * rather than a field: the ports do not exist until the stack has started. - * - *

The TCK owns the provider lifecycle from here — it registers the provider with the - * OpenFeature API under a scenario-scoped domain, waits for it to become ready, and shuts it - * down afterwards. Do not call {@code setProvider} or {@code initialize} yourself. + *

Called after {@link #startSuite()}, so an implementation may build it there and return the + * same instance on every call. It must not be {@code null}. * - * @param endpoint host and mapped ports of the running backend stack - * @return a configured, uninitialised provider + * @return the backend control for this suite */ - FeatureProvider createProvider(BackendEndpoint endpoint); + BackendControl backendControl(); /** * Creates a provider pointed at a backend that does not exist. @@ -100,15 +89,26 @@ public interface ProviderTckHarness { * reach its backend settles into {@code ERROR} and emits {@code PROVIDER_ERROR} rather than * hanging or throwing out of {@code setProvider}. * - *

Point this at a closed port on localhost. Do not point it at the Compose stack — the stack - * must stay up and reachable, and simulated outages belong to the control API. + *

Point this at a closed port on localhost. Do not point it at the backend under test — that + * must stay up and reachable, and simulated outages belong to {@link BackendControl}. * *

Configure a short connection deadline. The scenario allows a bounded time for the error * event to arrive, and a provider with a 30-second connect timeout will not make it. * + *

Defaults to throwing, because a provider with no backend has no way to be unreachable. + * Such a harness leaves {@link Capability#UNAVAILABLE_INIT} undeclared and the scenarios that + * would call this are reported as skipped, so the default is never reached. Reaching it means a + * capability was declared that the harness cannot back up. + * * @return a configured provider that cannot reach a backend */ - FeatureProvider createUnavailableProvider(); + default FeatureProvider createUnavailableProvider() { + throw new UnsupportedOperationException(getClass().getName() + " does not implement " + + "createUnavailableProvider(). This is a test-configuration bug rather than a provider " + + "defect: an @unavailable scenario ran, so the harness declared " + + "Capability.UNAVAILABLE_INIT without supplying a provider that cannot reach its " + + "backend. Remove that capability, or implement this method."); + } /** * Declares which optional parts of the provider contract this provider supports. @@ -126,53 +126,27 @@ default Set capabilities() { } /** - * Returns the Compose service name that hosts the control API and the backend the provider - * connects to. - * - * @return the Compose service name, {@code backend} by default - */ - default String backendService() { - return "backend"; - } - - /** - * Returns the container-internal port the control API listens on. - * - * @return the control API port, {@code 8080} by default - */ - default int controlPort() { - return 8080; - } - - /** - * Returns extra services and container-internal ports to expose, for stacks that contain more - * than the backend service. - * - *

Keys are Compose service names, values are container-internal ports. Resolve the mapped - * ports with {@link BackendEndpoint#port(String, int)}. + * Prepares whatever must exist before the first scenario — a container stack, a temporary + * directory, a local server. * - * @return additional services and ports to expose, empty by default - */ - default Map> additionalExposedPorts() { - return Collections.emptyMap(); - } - - /** - * Returns the control API configuration name used to seed the canonical flag set. + *

Called once, before any scenario, and always paired with {@link #stopSuite()}. Defaults to + * doing nothing, which is right for a harness whose backend is a data structure in this JVM. * - * @return the configuration name passed to {@code POST /start}, {@code default} by default + *

{@link #backendControl()} is called immediately afterwards, so this is where to build it + * if it needs something that only exists once the suite has started. */ - default String defaultConfig() { - return "default"; + default void startSuite() { + // Nothing to start by default. } /** - * Returns how long to wait for the Compose stack to become reachable. + * Releases whatever {@link #startSuite()} created. * - * @return the stack startup timeout, 60 seconds by default + *

Called once, after the last scenario, and also if suite startup fails partway through, so + * it must tolerate being called when startup did not complete. */ - default Duration startupTimeout() { - return Duration.ofSeconds(60); + default void stopSuite() { + // Nothing to stop by default. } /** @@ -184,8 +158,8 @@ default Duration startupTimeout() { * interval before it notices. Set this to comfortably exceed your worst-case detection latency, * or the suite will report timeouts that are really just impatience. * - *

Individual scenarios can tighten this with the explicit - * {@code within {int}ms} step, which always wins over this value. + *

Individual scenarios can tighten this with the explicit {@code within {int}ms} step, which + * always wins over this value. * * @return the default event await timeout, 12 seconds by default */ @@ -201,17 +175,4 @@ default Duration eventTimeout() { default Duration readyTimeout() { return Duration.ofSeconds(30); } - - /** - * Returns how long to pause after a control API call before continuing. - * - *

Covers the gap between the control API acknowledging a command and the backend actually - * having acted on it. Raise it if you see flakiness immediately after - * {@code the flag was modified} or a provider setup step. - * - * @return the settle time, 50 milliseconds by default - */ - default Duration settleTime() { - return Duration.ofMillis(50); - } } diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java similarity index 52% rename from tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java rename to tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java index 4f3304d94..6083afc56 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java @@ -13,27 +13,38 @@ * infrastructure at all. The canonical feature files are packaged inside this JAR and selected from * the classpath, so consumers need no git submodule of their own. * - *

To adopt the TCK, extend this class, implement the four abstract methods of - * {@link ProviderTckHarness}, and register the concrete class in - * {@code src/test/resources/META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness}. + *

Which base class to extend

+ * + *

This one is lifecycle-agnostic: it starts nothing and knows nothing about how the backend is + * reached. Extend it directly when your provider has no backend — an in-memory, + * environment-variable or file-based provider — and supply an in-process {@link BackendControl}. + * + *

When your provider talks to an external backend, extend {@link ContainerizedProviderTckTest} + * instead. It adds the Compose stack lifecycle, port discovery and {@link HttpBackendControl}, and + * the HTTP control API in {@code openapi/control-api.yaml} remains the normative contract for that + * conformance claim. In-process control is for backend-less providers only; an external backend + * driven through a custom in-JVM {@code BackendControl} bypasses that contract and proves nothing. + * + *

Serial execution

* *

Scenarios run serially, and this class enforces that rather than merely - * asking for it. Control API state — which flags are seeded, whether the backend is reachable — is - * global to the Compose stack, so concurrent scenarios corrupt each other: one scenario's - * {@code /start} restarts the backend underneath another's disconnect assertion. The failure looks - * like a flaky provider rather than a broken test, which makes it expensive to diagnose. + * asking for it. Backend state — which flags are seeded, whether the backend is reachable — is + * global to the suite, so concurrent scenarios corrupt each other: one scenario's reconnect + * restarts the backend underneath another's disconnect assertion. The failure looks like a flaky + * provider rather than a broken test, which makes it expensive to diagnose. * *

The suite therefore pins {@code cucumber.execution.parallel.enabled=false} here, where it * overrides any {@code junit-platform.properties} the consuming module happens to ship. Several * providers already enable Cucumber parallelism for their own suites, and inheriting that setting * silently breaks the TCK. * - *

Note this class carries no lifecycle code. The Compose stack, the control API client, provider - * registration and event awaiting are all owned by the step definitions in - * {@code dev.openfeature.contrib.tools.providertck.steps}, which reach the harness through - * {@link TckRuntime}. + *

Note this class carries no lifecycle code of its own. Provider registration, event awaiting + * and backend manipulation are owned by the step definitions in + * {@code dev.openfeature.contrib.tools.providertck.steps}, which reach the harness and its + * {@link BackendControl} through {@link TckRuntime}. * * @see ProviderTckHarness + * @see ContainerizedProviderTckTest */ @Suite @IncludeEngines("cucumber") @@ -43,4 +54,4 @@ @ConfigurationParameter(key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, value = "same_thread") @ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.tools.providertck.steps") @ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -public abstract class AbstractProviderTckTest implements ProviderTckHarness {} +public abstract class ProviderTckTest implements ProviderTckHarness {} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java index 82da5aa11..cd0dccb88 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java @@ -1,37 +1,31 @@ package dev.openfeature.contrib.tools.providertck; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.File; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.ServiceLoader; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.ComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; /** - * Suite-scoped runtime: discovers the provider's harness, owns the Compose stack, and exposes the - * control API client to the step definitions. + * Suite-scoped runtime: discovers the provider's harness, drives its suite lifecycle, and exposes + * its {@link BackendControl} to the step definitions. * - *

The Compose stack is started once, before the first scenario, and stopped - * after the last one. It is never stopped or restarted in between. Testcontainers cannot reliably - * preserve dynamically mapped host ports across a container restart, so a restart would silently - * invalidate every provider already pointed at the old port. Backend unavailability is therefore - * always simulated inside the running stack through the control API. See the normative statement of - * this invariant in {@code openapi/control-api.yaml}. + *

This class knows nothing about containers, ports or transports. Whatever must exist before the + * first scenario is created by {@link ProviderTckHarness#startSuite()} and released by + * {@link ProviderTckHarness#stopSuite()} — a Compose stack for + * {@link ContainerizedProviderTckTest}, nothing at all for a harness whose backend is a data + * structure in this JVM. + * + *

The lifecycle runs once: started before the first scenario, stopped after the + * last one, never cycled in between. Scenario isolation is achieved through + * {@link BackendControl#prepareScenario()} instead. * *

State is static because Cucumber's {@code @BeforeAll} / {@code @AfterAll} hooks are static and - * the stack must outlive individual scenarios. Consequently only one TCK suite may run per JVM fork - * at a time. + * the runtime must outlive individual scenarios. Consequently only one TCK suite may run per JVM + * fork at a time. */ -@SuppressFBWarnings( - value = "EI_EXPOSE_REP", - justification = "The harness and control API client are shared collaborators by design; " - + "step definitions must act on the same instances the suite started") public final class TckRuntime { private static final Logger log = LoggerFactory.getLogger(TckRuntime.class); @@ -42,42 +36,52 @@ public final class TckRuntime { private static TckRuntime instance; private final ProviderTckHarness harness; - private final ComposeContainer compose; - private final ControlApiClient controlApi; - private final BackendEndpoint endpoint; + private final BackendControl backendControl; - private TckRuntime(ProviderTckHarness harness, ComposeContainer compose) { + private TckRuntime(ProviderTckHarness harness, BackendControl backendControl) { this.harness = harness; - this.compose = compose; - this.endpoint = new BackendEndpoint(compose, harness.backendService()); - String baseUrl = "http://" + compose.getServiceHost(harness.backendService(), null) + ":" - + compose.getServicePort(harness.backendService(), harness.controlPort()); - this.controlApi = new ControlApiClient(baseUrl, harness.settleTime()); + this.backendControl = backendControl; } /** - * Starts the Compose stack if it is not already running, and returns the shared runtime. + * Starts the suite lifecycle if it is not already running, and returns the shared runtime. * * @return the suite-scoped runtime */ public static synchronized TckRuntime startIfNeeded() { - if (instance == null) { - ProviderTckHarness harness = discoverHarness(); - log.info("Provider TCK harness: {}", harness.getClass().getName()); - instance = new TckRuntime(harness, startCompose(harness)); - instance.controlApi.awaitReady(harness.startupTimeout()); - log.info("Control API ready at {}", instance.controlApi.baseUrl()); + if (instance != null) { + return instance; + } + ProviderTckHarness harness = discoverHarness(); + log.info("Provider TCK harness: {}", harness.getClass().getName()); + + harness.startSuite(); + try { + BackendControl control = harness.backendControl(); + if (control == null) { + throw new IllegalStateException(harness.getClass().getName() + + ".backendControl() returned null. Every harness must supply the seam through " + + "which the TCK manipulates the backend — HttpBackendControl for an external " + + "backend, an in-process implementation for a provider that has none."); + } + log.info("Backend control: {}", control.description()); + instance = new TckRuntime(harness, control); + } catch (RuntimeException e) { + // startSuite() may have allocated a container stack before this failed. + harness.stopSuite(); + throw e; } return instance; } /** - * Stops the Compose stack and releases the shared runtime. + * Runs the harness's suite teardown and releases the shared runtime. */ public static synchronized void stop() { if (instance != null) { - instance.compose.stop(); + ProviderTckHarness harness = instance.harness; instance = null; + harness.stopSuite(); } } @@ -85,7 +89,7 @@ public static synchronized void stop() { * Returns the running runtime. * * @return the suite-scoped runtime - * @throws IllegalStateException if the stack has not been started + * @throws IllegalStateException if the suite has not been started */ public static synchronized TckRuntime get() { if (instance == null) { @@ -104,46 +108,12 @@ public ProviderTckHarness harness() { } /** - * Returns the client for the backend's control API. - * - * @return the control API client - */ - public ControlApiClient controlApi() { - return controlApi; - } - - /** - * Returns the host and mapped ports of the running stack. + * Returns the seam through which the TCK manipulates the backend. * - * @return the backend endpoint + * @return the backend control for this suite */ - public BackendEndpoint endpoint() { - return endpoint; - } - - private static ComposeContainer startCompose(ProviderTckHarness harness) { - File composeFile = harness.composeFile(); - if (!composeFile.isFile()) { - throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() - + ". ProviderTckHarness.composeFile() is resolved relative to the module directory."); - } - ComposeContainer compose = new ComposeContainer(composeFile); - - compose.withExposedService(harness.backendService(), harness.controlPort(), Wait.forListeningPort()); - for (Integer port : harness.backendPorts()) { - compose.withExposedService(harness.backendService(), port, Wait.forListeningPort()); - } - for (Map.Entry> service : - harness.additionalExposedPorts().entrySet()) { - for (Integer port : service.getValue()) { - compose.withExposedService(service.getKey(), port, Wait.forListeningPort()); - } - } - compose.withStartupTimeout(harness.startupTimeout()); - - log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); - compose.start(); - return compose; + public BackendControl backendControl() { + return backendControl; } /** @@ -169,7 +139,8 @@ private static ProviderTckHarness discoverHarness() { if (found.isEmpty()) { throw new IllegalStateException("No ProviderTckHarness found. Write a test class extending " - + "AbstractProviderTckTest; it is both the JUnit suite and the harness."); + + "ContainerizedProviderTckTest (external backend) or ProviderTckTest (no backend); " + + "it is both the JUnit suite and the harness."); } if (found.size() == 1) { return found.get(0); diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java index 526068cb4..6a3ff9cf6 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java @@ -1,5 +1,6 @@ package dev.openfeature.contrib.tools.providertck.steps; +import dev.openfeature.contrib.tools.providertck.BackendControl; import dev.openfeature.contrib.tools.providertck.ProviderTckHarness; import dev.openfeature.contrib.tools.providertck.TckRuntime; import dev.openfeature.contrib.tools.providertck.TckState; @@ -7,8 +8,13 @@ /** * Base for the TCK step definition classes. * - *

Holds the PicoContainer-injected scenario state and gives subclasses convenience access to the - * suite-scoped runtime. + *

Holds the PicoContainer-injected scenario state and gives subclasses the only two collaborators + * a step is allowed to reach: the provider author's harness, and the {@link BackendControl} that + * manipulates the backend. + * + *

Deliberately no accessor for the runtime itself. Steps must not know whether the backend is a + * container reached over HTTP or a map in this JVM — that is exactly what {@link BackendControl} + * exists to hide, and it is what lets the same Gherkin run in both modes. */ public abstract class AbstractSteps { @@ -20,20 +26,20 @@ protected AbstractSteps(TckState state) { } /** - * Returns the suite-scoped runtime that owns the Compose stack and control API. + * Returns the provider author's harness. * - * @return the running TCK runtime + * @return the discovered harness */ - protected TckRuntime runtime() { - return TckRuntime.get(); + protected ProviderTckHarness harness() { + return TckRuntime.get().harness(); } /** - * Returns the provider author's harness. + * Returns the seam through which the backend is manipulated. * - * @return the discovered harness + * @return the backend control for this suite */ - protected ProviderTckHarness harness() { - return TckRuntime.get().harness(); + protected BackendControl backend() { + return TckRuntime.get().backendControl(); } } diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java index 8fe239414..35652f943 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java @@ -19,6 +19,7 @@ import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; +import java.time.Duration; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -27,9 +28,13 @@ import org.slf4j.LoggerFactory; /** - * Lifecycle and control API steps: bringing the Compose stack up, gating scenarios on declared + * Lifecycle and backend-control steps: bringing the suite up, gating scenarios on declared * capabilities, creating and registering the provider under test, and simulating backend outages. * + *

Every step that touches the backend goes through {@link #backend()}. Nothing here knows whether + * that is a container driven over HTTP or an in-memory provider manipulated directly, which is what + * lets one set of feature files cover both. + * *

Step vocabulary is inherited from the flagd test harness so that existing feature files port * with a near-zero diff. The only change is dropping the word {@code flagd} from the provider setup * step: {@code a stable flagd provider} becomes {@code a stable provider}. @@ -43,7 +48,7 @@ public ProviderSteps(TckState state) { } /** - * Starts the Compose stack once, before the first scenario. + * Runs the harness's suite startup once, before the first scenario. */ @BeforeAll public static void beforeAll() { @@ -51,7 +56,7 @@ public static void beforeAll() { } /** - * Stops the Compose stack after the last scenario. + * Runs the harness's suite teardown after the last scenario. */ @AfterAll public static void afterAll() { @@ -66,6 +71,11 @@ public static void afterAll() { * configuration-change events should see those scenarios visibly excluded, never silently * green. * + *

This is also how a backend with no connection to lose stays honest. A harness whose + * {@link dev.openfeature.contrib.tools.providertck.BackendControl} cannot simulate an outage + * leaves {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT} undeclared, and the + * scenarios needing them are skipped here — before any step can reach an unsupported operation. + * * @param scenario the scenario about to run */ @Before(order = 0) @@ -81,19 +91,18 @@ public void gateOnCapabilities(Scenario scenario) { } /** - * Restores the backend to a running, freshly seeded state before each scenario. + * Restores the backend to the state every scenario starts from. * - *

Scenario isolation is achieved here, through the control API, and never by restarting - * containers — see the no-container-restart invariant in {@code openapi/control-api.yaml}. + *

Scenario isolation is achieved here and nowhere else — never by restarting containers, and + * never by relying on scenarios happening not to interfere. */ @Before(order = 10) public void prepareBackend() { - ProviderTckHarness harness = harness(); - runtime().controlApi().prepareScenario(harness.defaultConfig()); + backend().prepareScenario(); } /** - * Tears the provider down without disturbing the Compose stack. + * Tears the provider down without disturbing the backend. * *

Replaces the domain's provider with a {@link NoOpProvider} through the SDK lifecycle rather * than calling {@code shutdown()} directly, because only the former makes the SDK detach the @@ -111,10 +120,10 @@ public void tearDown() { * Creates the provider under test and registers it under a scenario-scoped domain. * *

Two provider flavours are recognised. A {@code stable} provider is built by the harness - * against the running stack and registered with {@code setProviderAndWait}, so the step does not - * return until the provider is ready. An {@code unavailable} provider points at a dead backend - * and is registered with {@code setProvider}, deliberately without waiting — the scenario's - * whole point is that readiness never arrives. + * against the running backend and registered with {@code setProviderAndWait}, so the step does + * not return until the provider is ready. An {@code unavailable} provider points at a dead + * backend and is registered with {@code setProvider}, deliberately without waiting — the + * scenario's whole point is that readiness never arrives. * * @param flavour either {@code stable} or {@code unavailable} */ @@ -126,7 +135,7 @@ public void createProvider(String flavour) { switch (flavour) { case "stable": - provider = harness.createProvider(runtime().endpoint()); + provider = harness.createProvider(); waitForReady = true; break; case "unavailable": @@ -161,7 +170,7 @@ public void createProvider(String flavour) { */ @When("the connection is lost") public void theConnectionIsLost() { - runtime().controlApi().stop(); + backend().disconnect(); } /** @@ -171,7 +180,7 @@ public void theConnectionIsLost() { */ @When("the connection is lost for {int}s") public void theConnectionIsLostFor(int seconds) { - runtime().controlApi().restart(seconds); + backend().disconnectFor(Duration.ofSeconds(seconds)); } /** @@ -184,7 +193,7 @@ public void theConnectionIsLostFor(int seconds) { */ @When("the connection is restored") public void theConnectionIsRestored() { - runtime().controlApi().start(harness().defaultConfig()); + backend().reconnect(); } /** @@ -192,7 +201,7 @@ public void theConnectionIsRestored() { */ @When("the flag was modified") public void theFlagWasModified() { - runtime().controlApi().change(); + backend().changeFlag(); } /** From 61309e35511415511ac60bb97cd05970a0d852c5 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 11:19:18 +0200 Subject: [PATCH 2/3] feat(provider-tck): in-process backend control and an in-memory self-test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Providers without an external backend — in-memory, environment-variable, file-based — could not run the TCK: every path to the backend went through Docker, Compose and HTTP. Add the in-process control path so they can, and use it to give the TCK a self-test. InProcessBackendControl manipulates the SDK's InMemoryProvider directly. Flag operations are map updates and a configuration change is updateFlag(), so the event the suite awaits is the provider's own PROVIDER_CONFIGURATION_CHANGED rather than one the TCK synthesised. It is deliberately bound to InMemoryProvider and deliberately not a general-purpose escape hatch: an external backend driven through a side channel bypasses the HTTP control API, which is the only thing that makes a conformance claim portable across languages. The README and the BackendControl javadoc say so explicitly. Connection control is modelled through the existing capability mechanism rather than no-op stubs. disconnect(), reconnect() and disconnectFor() are left at their throwing defaults, and the harness leaves STALE and UNAVAILABLE_INIT undeclared, so those scenarios are reported as skipped-with-reason. Over-declaring a capability the control cannot back fails loudly with a message naming the fix — an UnsupportedOperationException reached from a live scenario is a test-configuration bug, never a skip. InProcessBackendControlTest pins that, because a scenario that never runs cannot prove it would have failed. InMemoryProviderTckTest runs the full applicable suite against InMemoryProvider: 26 passed, 3 skipped by capability, no Docker, under a second. It is both the reference adoption for a backend-less provider and a CI canary that reports a broken step definition or capability gate in seconds — wired as its own Docker-free job alongside the existing matrix, which is unchanged. Signed-off-by: Simon Schrottner --- .github/workflows/ci.yml | 36 +++ tools/provider-tck/README.md | 116 ++++++++- tools/provider-tck/pom.xml | 14 ++ .../tools/providertck/BackendControl.java | 13 +- .../contrib/tools/providertck/Capability.java | 15 ++ .../providertck/InProcessBackendControl.java | 226 ++++++++++++++++++ .../tools/providertck/ProviderTckHarness.java | 2 +- .../tools/providertck/ProviderTckTest.java | 3 +- .../contrib/tools/providertck/TckRuntime.java | 2 +- .../providertck/InMemoryProviderTckTest.java | 78 ++++++ .../InProcessBackendControlTest.java | 104 ++++++++ 11 files changed, 593 insertions(+), 16 deletions(-) create mode 100644 tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java create mode 100644 tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java create mode 100644 tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fbdc4604..769d4216d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,42 @@ on: - main jobs: + # Fast canary for the Provider TCK: runs the full applicable conformance suite against the + # SDK's InMemoryProvider with no Docker, no Compose stack and no network. It finishes in + # seconds, so a broken step definition, a mis-wired capability gate or a regression in the + # shared harness is reported long before the containerised provider suites in `main` get + # there — and it points at the TCK rather than at whichever provider noticed first. + # + # Deliberately not a gate on `main`: the two run in parallel so a green run is not delayed. + # The same suite also runs inside `main` as part of the reactor build; this job exists to + # report it fast and in isolation. + provider-tck: + name: Provider TCK (no Docker) + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + # No submodules: this module's feature files, flags and control-API spec are in-repo. + + - name: Set up JDK 21 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5 + with: + java-version: 21 + distribution: 'temurin' + cache: maven + + - name: Cache local Maven repository + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}21-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}21-maven- + + - name: Verify the TCK against the in-memory provider + # No `e2e` profile and no Docker: the in-memory suite is not gated behind either. + run: mvn --batch-mode --activate-profiles codequality -pl tools/provider-tck -am clean verify + main: strategy: matrix: diff --git a/tools/provider-tck/README.md b/tools/provider-tck/README.md index 6c14dd0b2..7370f085f 100644 --- a/tools/provider-tck/README.md +++ b/tools/provider-tck/README.md @@ -23,7 +23,8 @@ contract" is an unverified claim. This is the shared suite that makes it checkab ``` -Requires Java 11+, JUnit 5, and a working Docker daemon. +Requires Java 11+ and JUnit 5. A working Docker daemon is needed only for providers with an +external backend — see [Which base class to extend](#which-base-class-to-extend). ### OpenFeature SDK compatibility @@ -55,8 +56,94 @@ uses only long-stable API — `OpenFeatureAPI`, `Client`, typed evaluation, `Pro - the provider↔backend wire protocol. How you talk to your backend is your business. - SDK behaviour. That belongs to the SDK's own test suite. +## Which base class to extend + +Two, and the choice is made by one question: **does your provider talk to something outside the +JVM?** + +| | Extend | Backend control | You supply | +|---|---|---|---| +| Provider has an external backend | `ContainerizedProviderTckTest` | `HttpBackendControl`, over the HTTP control API | a Compose stack, a control API, a test class | +| Provider has no backend — in-memory, environment variables, a local file | `ProviderTckTest` | an in-process `BackendControl` | a test class | + +`ContainerizedProviderTckTest` is the normal case and everything in [Adopting it](#adopting-it) +below describes it. It extends `ProviderTckTest` and adds the Compose lifecycle, port discovery and +control API client on top. + +### In-process control is for backend-less providers only + +Step definitions never touch a backend directly. They go through one interface, `BackendControl`, +which is what lets the same Gherkin run against a container over HTTP and against an in-memory +provider manipulated in the same JVM. + +That seam is not an invitation to skip the control API. **If your provider has an external backend, +use `HttpBackendControl` via `ContainerizedProviderTckTest`.** The control API described in +[`openapi/control-api.yaml`](src/main/resources/openapi/control-api.yaml) is the normative contract +for those providers, and it is the whole basis of a portable conformance claim: another language's +TCK drives the same endpoints against the same stack and must get the same answers. + +A custom in-JVM `BackendControl` that reaches an external backend through a side channel — a +test-only admin client, a shared database handle, a static hook inside the provider — bypasses that +contract. It will pass, and it will prove nothing, because the path it exercised is not the path the +contract describes. + +In-process control exists for providers that have **nothing to contract with**, where "the backend" +is a data structure in the same JVM. For those, flag operations are map updates and a configuration +change is the provider's own update mechanism emitting its own event. + +### Adopting it without a backend + +`InProcessBackendControl` implements this for the SDK's `InMemoryProvider`, seeded with the +canonical flag set. The entire adoption is three methods: + +```java +public class MyProviderTckTest extends ProviderTckTest { + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING); + } +} +``` + +One object backs both factory methods because in-process the flag store and the provider are the +same thing: `changeFlag()` has to reach the live provider instance to emit an event from it. + +**Connection control does not apply**, and the capability declaration is where you say so rather +than stubbing it out. An in-memory provider has no connection to lose, so +`InProcessBackendControl` leaves `disconnect()`, `reconnect()` and `disconnectFor()` unimplemented — +they throw. Leaving `STALE` and `UNAVAILABLE_INIT` out of `capabilities()` is what keeps that +honest: the scenarios needing them are skipped before any step can reach an unsupported operation. + +Get that pairing wrong — declare `STALE` against a control that cannot disconnect — and you get an +`UnsupportedOperationException` naming the fix, not a silent pass. That is deliberate. A +`BackendControl` may throw `UnsupportedOperationException` for operations it does not support, and +reaching one from a scenario that actually ran is a **test-configuration bug**, never a skip. + +The TCK's own self-test is exactly this class: see +[`InMemoryProviderTckTest`](src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java), +which runs the full applicable suite against `InMemoryProvider` with no Docker in well under a +second. It doubles as the reference adoption and as the Docker-free CI canary. + ## Adopting it +This section describes a provider with an external backend — the common case. Four things to implement, then two small files. ### 1. A Docker Compose stack @@ -130,7 +217,7 @@ Two details are load-bearing: ### 4. The test class ```java -public class MyProviderTckTest extends AbstractProviderTckTest { +public class MyProviderTckTest extends ContainerizedProviderTckTest { @Override public File composeFile() { @@ -172,7 +259,7 @@ registration, no system property, no build configuration. Each class is its own own Compose stack, and they can share a base class: ```java -abstract class AbstractMyProviderTckTest extends AbstractProviderTckTest { +abstract class AbstractMyProviderTckTest extends ContainerizedProviderTckTest { protected abstract Mode mode(); // composeFile(), createProvider(), capabilities() ... shared here } @@ -218,10 +305,10 @@ green on scenarios it did not run is worse than no suite at all. | Capability | Tag | Meaning | |---|---|---| | `EVENTS` | `@events` | emits lifecycle events at all | -| `STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | +| `STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss — *needs connection control* | | `CONFIGURATION_CHANGE` | `@configuration-change` | detects config changes, emits `PROVIDER_CONFIGURATION_CHANGED` | | `OBJECT` | `@object` | supports structured flag values | -| `UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging on a dead backend | +| `UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging on a dead backend — *needs connection control* | | `STRICT_NUMERIC_TYPING` | `@strict-numeric-typing` | does not coerce between integer and float | | `TARGETING` | `@targeting` | reserved, no scenarios yet | | `CACHING` | `@caching` | reserved, no scenarios yet | @@ -229,6 +316,12 @@ green on scenarios it did not run is worse than no suite at all. The default is every capability. **Narrow it, do not widen it**: start from the default, run the suite, and remove only what your provider genuinely cannot do. +`STALE` and `UNAVAILABLE_INIT` are the two that need a backend the provider can be cut off from. +They are what a backend-less provider leaves undeclared — see +[In-process control is for backend-less providers only](#in-process-control-is-for-backend-less-providers-only). +Declaring one against a `BackendControl` that cannot simulate an outage fails the scenario with an +`UnsupportedOperationException` naming the fix, rather than passing it. + ```java @Override public Set capabilities() { @@ -254,8 +347,8 @@ needs most of a poll interval. Every await timeout is therefore overridable. |---|---|---| | `eventTimeout()` | 12s | waiting for a provider event | | `readyTimeout()` | 30s | waiting for a provider to reach a lifecycle state | -| `startupTimeout()` | 60s | bringing the Compose stack up | -| `settleTime()` | 50ms | pause after a control API call | +| `startupTimeout()` | 60s | bringing the Compose stack up (`ContainerizedProviderTckTest` only) | +| `settleTime()` | 50ms | pause after a control API call (`ContainerizedProviderTckTest` only) | ```java @Override @@ -274,6 +367,9 @@ use the explicit `within {int}ms` step, which always wins. mvn test -Dtest=MyProviderTckTest ``` +A suite extending `ProviderTckTest` with in-process control needs no Docker and no network. A suite +extending `ContainerizedProviderTckTest` needs a working Docker daemon for its Compose stack. + Scenarios run **serially** and the suite enforces this, overriding any `cucumber.execution.parallel.enabled=true` in your module's `junit-platform.properties`. Control API state is global to the Compose stack, so concurrent scenarios corrupt each other — one scenario's @@ -335,6 +431,12 @@ consumers — the features stay on the classpath and stay inside the JAR. `@targeting` tag is reserved for context-passthrough scenarios once the gap above is closed. - **Caching.** Whether a stale provider keeps serving last-known values during an outage depends on whether it holds a local copy of the ruleset. The `@caching` tag is reserved; no scenarios yet. +- **Setting and removing individual flags.** `BackendControl` exposes `prepareScenario()` and + `changeFlag()` — reset to the canonical baseline, and mutate `changing-flag` — because those are + what the Gherkin needs and what the control API defines. Finer-grained `setFlag(key, value)` / + `removeFlag(key)` operations would need control API endpoints that do not exist yet, so adding + them to the interface would produce methods `HttpBackendControl` could not implement. They belong + to a control API revision, not to the Java seam. - **Hooks.** Not covered. - **Flag metadata.** The flagd harness has metadata scenarios; they are not yet ported. - **Multi-suite JVMs.** `TckRuntime` is static, so TCK suites run one at a time within a JVM fork. diff --git a/tools/provider-tck/pom.xml b/tools/provider-tck/pom.xml index a92c7c715..ec59f7a08 100644 --- a/tools/provider-tck/pom.xml +++ b/tools/provider-tck/pom.xml @@ -182,6 +182,20 @@ slf4j-api ${slf4j.version} + + + + org.junit.jupiter + junit-jupiter + + test + diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java index 820ffdaa8..427f80d36 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java @@ -6,9 +6,10 @@ * The single seam between the TCK's step definitions and whatever manipulates the backend. * *

Step definitions never talk to a backend directly. They talk to this interface, which is why - * the same Gherkin can run unchanged against any backend an implementation can drive — - * a containerised one over HTTP ({@link HttpBackendControl}) being the first. Nothing below this - * line knows about ports, containers or transports. + * the same Gherkin runs unchanged against a containerised backend driven over HTTP + * ({@link HttpBackendControl}) and against a provider manipulated in-process + * ({@link InProcessBackendControl}). Nothing below this line knows about ports, containers or + * transports. * *

Which implementation is right for your provider

* @@ -23,9 +24,9 @@ * static hook inside the provider. It will pass, and it will prove nothing, because the thing it * exercised is not the thing the contract describes. * - *

An in-JVM implementation is legitimate only for providers that have no backend to - * contract with: in-memory, environment-variable and file-based providers, where "the backend" is a - * data structure in the same JVM. + *

In-process control exists for providers that have no backend to contract with: + * in-memory, environment-variable and file-based providers, where "the backend" is a data structure + * in the same JVM. See {@link InProcessBackendControl}. * *

Operations a backend may not support

* diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java index 8203db0d8..c1e9a4c0d 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java @@ -17,6 +17,21 @@ * never as passed. Silently green scenarios would make a conformance suite worthless. * *

Scenarios with no capability tag are considered mandatory and always run. + * + *

The connection-dependent capabilities

+ * + *

{@link #STALE} and {@link #UNAVAILABLE_INIT} are the two that require a backend the provider + * can be cut off from. They are what a harness leaves undeclared when its {@link BackendControl} + * has no connection to control — an in-memory, environment-variable or file-based provider, where + * the backend is a data structure in the same JVM. Every step that would call + * {@link BackendControl#disconnect()}, {@link BackendControl#reconnect()} or + * {@link ProviderTckHarness#createUnavailableProvider()} lives in a scenario carrying one of these + * two tags, so undeclaring them skips those scenarios before an unsupported operation can be + * reached. + * + *

Getting that pairing wrong surfaces as an {@link UnsupportedOperationException} rather than a + * skip, which is deliberate: it means a capability was declared that the harness cannot back up, + * and that is a test-configuration bug. */ public enum Capability { diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java new file mode 100644 index 000000000..91ef2fb6f --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java @@ -0,0 +1,226 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.MutableStructure; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * {@link BackendControl} that manipulates the SDK's {@link InMemoryProvider} directly, with no + * backend, no container and no HTTP. + * + *

This exists so that providers with nothing to connect to — in-memory, environment-variable and + * file-based providers — can run the TCK. For those, "the backend" is a data structure in the same + * JVM: seeding flags is building a map, and changing one is + * {@link InMemoryProvider#updateFlag(String, Flag)}, which emits + * {@code PROVIDER_CONFIGURATION_CHANGED} through the provider's own event mechanism rather than + * through a simulated one. + * + *

This is not a shortcut for providers that do have a backend. Reaching into an + * external backend from inside the JVM — a test-only admin client, a shared database handle, a + * static hook in the provider — produces a suite that passes while proving nothing, because the + * path it exercised is not the path the contract describes. Those providers use + * {@link HttpBackendControl} via {@link ContainerizedProviderTckTest}, and the control API in + * {@code openapi/control-api.yaml} stays the normative contract. See {@link BackendControl}. + * + *

Connection control

+ * + *

{@link #disconnect()}, {@link #reconnect()} and {@link #disconnectFor} are not implemented, so + * they inherit the interface defaults and throw. An in-memory provider has no connection to lose, + * and pretending otherwise with a no-op would report {@code @stale} scenarios as passed. The + * harness instead leaves {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT} + * undeclared, and those scenarios are reported as skipped. + * + *

Ownership of the provider

+ * + *

This class both seeds the flags and creates the provider that serves them, because in-process + * they are the same object: {@link #changeFlag()} has to reach the live provider instance to emit + * an event from it. A harness therefore wires both of its factory methods to one instance: + * + * 

{@code
+ * private final InProcessBackendControl control = new InProcessBackendControl();
+ *
+ * @Override
+ * public BackendControl backendControl() {
+ *     return control;
+ * }
+ *
+ * @Override
+ * public FeatureProvider createProvider() {
+ *     return control.createProvider();
+ * }
+ * }
+ */ +public final class InProcessBackendControl implements BackendControl { + + /** The flag {@link #changeFlag()} mutates, as defined by {@code flags/canonical-flags.json}. */ + private static final String CHANGING_FLAG = "changing-flag"; + + private static final String CHANGING_BASELINE = "foo"; + private static final String CHANGING_CHANGED = "bar"; + + /** + * The canonical flag set, never mutated after construction. + * + *

Scenario isolation depends on that: {@link InMemoryProvider} copies the map it is given, + * and {@code updateFlag} writes only to the provider's copy, so every provider handed out by + * {@link #createProvider()} starts from an untouched baseline. + */ + private final Map> baseline = canonicalFlags(); + + /** The provider serving the current scenario, or {@code null} between scenarios. */ + private InMemoryProvider current; + + /** Which variant {@code changing-flag} currently resolves to. */ + private String changingVariant = CHANGING_BASELINE; + + /** + * Creates the provider for the scenario about to run, seeded with the canonical flag set. + * + *

Each call returns a fresh instance over a fresh copy of the baseline, which is what makes + * {@link #prepareScenario()} nothing more than dropping the previous reference. + * + * @return a configured, uninitialised in-memory provider + */ + @SuppressFBWarnings( + value = "EI_EXPOSE_REP", + justification = "Handing out the live provider is the contract, not a leak: in-process " + + "the flag store and the provider are one object, and changeFlag() must reach " + + "the same instance the TCK registered in order to emit an event from it") + public InMemoryProvider createProvider() { + changingVariant = CHANGING_BASELINE; + current = new InMemoryProvider(new HashMap<>(baseline)); + return current; + } + + @Override + public String description() { + return "in-process control of " + InMemoryProvider.class.getSimpleName(); + } + + /** + * {@inheritDoc} + * + *

Drops the reference to the previous scenario's provider. That is the whole reset: the + * baseline map is never mutated, so the {@link #createProvider()} call that follows produces a + * provider already at the baseline. Clearing the reference rather than leaving it dangling + * means a scenario that manipulates flags without creating a provider fails with a clear + * message instead of mutating a provider that has already been shut down. + */ + @Override + public void prepareScenario() { + current = null; + } + + /** + * {@inheritDoc} + * + *

Flips {@code changing-flag} between its two variants through + * {@link InMemoryProvider#updateFlag(String, Flag)}, so the event the suite awaits is the + * provider's own {@code PROVIDER_CONFIGURATION_CHANGED} — carrying {@code changing-flag} in + * {@code flagsChanged} — and not a signal the TCK synthesised. + * + *

Alternating rather than assigning a fixed variant keeps repeated calls within one scenario + * meaningful; the suite asserts that the resolved value differs, not what it became. + */ + @Override + public void changeFlag() { + changingVariant = CHANGING_CHANGED.equals(changingVariant) ? CHANGING_BASELINE : CHANGING_CHANGED; + requireProvider().updateFlag(CHANGING_FLAG, changingFlag(changingVariant)); + } + + private InMemoryProvider requireProvider() { + if (current == null) { + throw new IllegalStateException("No in-memory provider exists for this scenario. In-process backend " + + "control manipulates the provider itself, so the scenario must create one — with " + + "'Given a stable provider' — before any step that changes flag state."); + } + return current; + } + + /** + * Builds the canonical flag set as {@link InMemoryProvider} flags. + * + *

Mirrors {@code flags/canonical-flags.json} entry for entry. The two load-bearing details + * from that file hold here too: {@code missing-flag} is absent, which is what the + * {@code FLAG_NOT_FOUND} scenario tests, and no flag carries a + * {@link dev.openfeature.sdk.providers.memory.ContextEvaluator}, so every evaluation reports + * reason {@code STATIC} as the feature files expect. + * + * @return the canonical flag set + */ + private static Map> canonicalFlags() { + Map> flags = new LinkedHashMap<>(); + + flags.put( + "boolean-flag", + Flag.builder() + .variant("on", true) + .variant("off", false) + .defaultVariant("on") + .build()); + + flags.put( + "string-flag", + Flag.builder() + .variant("greeting", "hi") + .variant("parting", "bye") + .defaultVariant("greeting") + .build()); + + flags.put( + "integer-flag", + Flag.builder() + .variant("one", 1) + .variant("ten", 10) + .defaultVariant("ten") + .build()); + + flags.put( + "float-flag", + Flag.builder() + .variant("tenth", 0.1) + .variant("half", 0.5) + .defaultVariant("half") + .build()); + + flags.put( + "object-flag", + Flag.builder() + .variant("empty", new Value(new MutableStructure())) + .variant( + "template", + new Value(new MutableStructure() + .add("showImages", true) + .add("title", "Check out these pics!") + .add("imagesPerPage", 100))) + .defaultVariant("template") + .build()); + + // A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario. + flags.put( + "wrong-flag", + Flag.builder() + .variant("one", "uno") + .variant("two", "dos") + .defaultVariant("one") + .build()); + + flags.put(CHANGING_FLAG, changingFlag(CHANGING_BASELINE)); + + return Collections.unmodifiableMap(flags); + } + + private static Flag changingFlag(String defaultVariant) { + return Flag.builder() + .variant(CHANGING_BASELINE, CHANGING_BASELINE) + .variant(CHANGING_CHANGED, CHANGING_CHANGED) + .defaultVariant(defaultVariant) + .build(); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java index 72dd14c10..b7b76115a 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java @@ -32,7 +32,7 @@ * 

{@code
  * public class MyProviderTckTest extends ProviderTckTest {
  *
- *     private final MyInProcessControl control = new MyInProcessControl();
+ *     private final InProcessBackendControl control = new InProcessBackendControl();
  *
  *     @Override
  *     public BackendControl backendControl() {
diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java
index 6083afc56..f3ac3f43a 100644
--- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java
+++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java
@@ -17,7 +17,8 @@
  *
  * 
This one is lifecycle-agnostic: it starts nothing and knows nothing about how the backend is
  * reached. Extend it directly when your provider has no backend — an in-memory,
- * environment-variable or file-based provider — and supply an in-process {@link BackendControl}.
+ * environment-variable or file-based provider — and supply an in-process {@link BackendControl}
+ * such as {@link InProcessBackendControl}.
  *
  * 
When your provider talks to an external backend, extend {@link ContainerizedProviderTckTest}
  * instead. It adds the Compose stack lifecycle, port discovery and {@link HttpBackendControl}, and
diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java
index cd0dccb88..4489f4bc3 100644
--- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java
+++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java
@@ -62,7 +62,7 @@ public static synchronized TckRuntime startIfNeeded() {
                 throw new IllegalStateException(harness.getClass().getName()
                         + ".backendControl() returned null. Every harness must supply the seam through "
                         + "which the TCK manipulates the backend — HttpBackendControl for an external "
-                        + "backend, an in-process implementation for a provider that has none.");
+                        + "backend, InProcessBackendControl for a provider that has none.");
             }
             log.info("Backend control: {}", control.description());
             instance = new TckRuntime(harness, control);
diff --git a/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java
new file mode 100644
index 000000000..3d8ac5e25
--- /dev/null
+++ b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java
@@ -0,0 +1,78 @@
+package dev.openfeature.contrib.tools.providertck;
+
+import dev.openfeature.sdk.FeatureProvider;
+import dev.openfeature.sdk.providers.memory.InMemoryProvider;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Runs the OpenFeature Provider TCK against the SDK's own {@link InMemoryProvider}.
+ *
+ * 
This is the TCK's self-test, and it earns its keep twice over.
+ *
+ * 
It is the reference adoption for a provider with no backend. Everything a
+ * file-based or environment-variable provider needs to write is here, and it is three methods: hand
+ * over a {@link BackendControl}, hand over a provider, and say which capabilities hold.
+ *
+ * 
It is also the Docker-free canary. Because it needs no container, no Compose
+ * stack and no network, it runs in seconds on any machine and in any CI job, which makes it the
+ * fast check that catches a broken step definition, a mis-wired capability gate or a regression in
+ * the shared harness long before the containerised suites get a chance to. When a change breaks
+ * both this and the flagd suite, this one tells you within seconds and points at the TCK rather
+ * than at a provider.
+ *
+ * 
Note what it does not do: it is not a licence for providers that have a backend to
+ * test themselves this way. See {@link BackendControl} for why.
+ */
+public class InMemoryProviderTckTest extends ProviderTckTest {
+
+    /**
+     * Both the flag store and the factory for the provider that serves it.
+     *
+     * 
In-process the two are the same thing — {@code changeFlag()} has to reach the live provider
+     * instance to emit an event from it — so one instance backs both harness methods below.
+     */
+    private final InProcessBackendControl control = new InProcessBackendControl();
+
+    @Override
+    public BackendControl backendControl() {
+        return control;
+    }
+
+    @Override
+    public FeatureProvider createProvider() {
+        return control.createProvider();
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * 
Four capabilities, and each omission is a fact about {@link InMemoryProvider} rather than a
+     * convenience:
+     *
+     * 
    
+     *   
  • {@link Capability#STALE} — omitted. There is no connection to lose, so the provider can
+     *       never go {@code STALE}. {@link InProcessBackendControl} leaves
+     *       {@link BackendControl#disconnect()} unimplemented for the same reason, and this omission
+     *       is what keeps the two consistent: the scenario is skipped before any step can reach the
+     *       unsupported operation.
+     *   
  • {@link Capability#UNAVAILABLE_INIT} — omitted. Initialisation cannot fail when there is
+     *       nothing to connect to, so
+     *       {@link ProviderTckHarness#createUnavailableProvider()} is left at its throwing default.
+     *   
  • {@link Capability#TARGETING} and {@link Capability#CACHING} — omitted because no
+     *       scenario carries their tags yet. Nothing is skipped by leaving them out today.
+     * 

+     *
+     * 
{@link Capability#STRICT_NUMERIC_TYPING} is declared, and that is worth stating
+     * plainly: {@link InMemoryProvider} refuses to narrow {@code float-flag} (0.5) to an integer and
+     * reports {@code TYPE_MISMATCH} instead. It is the reference behaviour the capability describes.
+     */
+    @Override
+    public Set capabilities() {
+        return EnumSet.of(
+                Capability.EVENTS,
+                Capability.CONFIGURATION_CHANGE,
+                Capability.OBJECT,
+                Capability.STRICT_NUMERIC_TYPING);
+    }
+}
diff --git a/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java
new file mode 100644
index 000000000..14df30bb3
--- /dev/null
+++ b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java
@@ -0,0 +1,104 @@
+package dev.openfeature.contrib.tools.providertck;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import dev.openfeature.sdk.ImmutableContext;
+import dev.openfeature.sdk.ProviderEvaluation;
+import dev.openfeature.sdk.providers.memory.InMemoryProvider;
+import java.time.Duration;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Guards the two properties of {@link InProcessBackendControl} that the Gherkin cannot assert
+ * about itself.
+ *
+ * 
The first is that unsupported operations fail loudly. The whole
+ * skipped-by-capability design collapses into false confidence if a connection operation quietly
+ * does nothing, and a scenario that never runs cannot prove that it would have failed. These tests
+ * call the operations directly.
+ *
+ * 
The second is that scenario isolation actually isolates. {@link InMemoryProviderTckTest} would
+ * still pass if {@code changeFlag()} leaked into the next scenario, because no scenario evaluates
+ * {@code changing-flag} before modifying it.
+ */
+class InProcessBackendControlTest {
+
+    @Test
+    @DisplayName("connection operations throw rather than silently doing nothing")
+    void connectionOperationsThrow() {
+        InProcessBackendControl control = new InProcessBackendControl();
+
+        // The message has to name the fix, because whoever hits this is looking at a red scenario
+        // that reads like a provider defect and is in fact a capability declared in error.
+        assertThatThrownBy(control::disconnect)
+                .isInstanceOf(UnsupportedOperationException.class)
+                .hasMessageContaining("does not support 'disconnect'")
+                .hasMessageContaining("test-configuration bug")
+                .hasMessageContaining("Capability.STALE");
+
+        assertThatThrownBy(control::reconnect)
+                .isInstanceOf(UnsupportedOperationException.class)
+                .hasMessageContaining("does not support 'reconnect'");
+
+        assertThatThrownBy(() -> control.disconnectFor(Duration.ofSeconds(1)))
+                .isInstanceOf(UnsupportedOperationException.class)
+                .hasMessageContaining("does not support 'disconnectFor'");
+    }
+
+    @Test
+    @DisplayName("changing a flag without a provider fails instead of being lost")
+    void changeFlagWithoutProviderThrows() {
+        InProcessBackendControl control = new InProcessBackendControl();
+        control.prepareScenario();
+
+        assertThatThrownBy(control::changeFlag)
+                .isInstanceOf(IllegalStateException.class)
+                .hasMessageContaining("Given a stable provider");
+    }
+
+    @Test
+    @DisplayName("changeFlag changes the resolved value and the next scenario starts from baseline")
+    void changeFlagIsVisibleAndDoesNotLeak() throws Exception {
+        InProcessBackendControl control = new InProcessBackendControl();
+
+        control.prepareScenario();
+        InMemoryProvider first = control.createProvider();
+        first.initialize(new ImmutableContext());
+        assertThat(resolveChangingFlag(first)).isEqualTo("foo");
+
+        control.changeFlag();
+        assertThat(resolveChangingFlag(first))
+                .as("changeFlag must actually change what the provider resolves, not merely emit an event")
+                .isEqualTo("bar");
+
+        // The next scenario must not inherit that change. The baseline map is shared between every
+        // provider this control hands out, so a mutation that reached it would leak forwards.
+        control.prepareScenario();
+        InMemoryProvider second = control.createProvider();
+        second.initialize(new ImmutableContext());
+        assertThat(resolveChangingFlag(second))
+                .as("each scenario starts from the canonical baseline")
+                .isEqualTo("foo");
+    }
+
+    @Test
+    @DisplayName("the canonical flag set omits missing-flag")
+    void missingFlagIsAbsent() throws Exception {
+        InProcessBackendControl control = new InProcessBackendControl();
+        InMemoryProvider provider = control.createProvider();
+        provider.initialize(new ImmutableContext());
+
+        // Absence is what the FLAG_NOT_FOUND scenario tests, so seeding it by accident would turn
+        // that scenario green for the wrong reason.
+        assertThatThrownBy(() -> provider.getStringEvaluation("missing-flag", "fallback", new ImmutableContext()))
+                .hasMessageContaining("missing-flag");
+    }
+
+    private static String resolveChangingFlag(InMemoryProvider provider) {
+        ProviderEvaluation evaluation =
+                provider.getStringEvaluation("changing-flag", "unset", new ImmutableContext());
+        return evaluation.getValue();
+    }
+}

From 6302455ec6eb83d62bed7a40626429ef6b69e1d7 Mon Sep 17 00:00:00 2001
From: Simon Schrottner 
Date: Mon, 24 Aug 2026 11:47:42 +0200
Subject: [PATCH 3/3] test(provider-tck): also run the self-test against
 MultiProvider
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A provider that delegates is still a provider, and delegation is where the
contract is easiest to drop on the floor: a variant that does not survive the
hop, a reason rewritten, an error code flattened, an event that never arrives.

MultiProviderTckTest runs the suite against the SDK's MultiProvider wrapping
exactly one InMemoryProvider. One child is the interesting configuration rather
than a degenerate one — the correct answer is then precisely what
InMemoryProviderTckTest already asserts, so any difference between the two suites
is attributable to MultiProvider and nothing else. This is not a test of
aggregation; it is a test that delegation is transparent.

It found something on the first run. MultiProvider extends EventProvider but
never subscribes to its children, so a child's PROVIDER_CONFIGURATION_CHANGED —
along with its PROVIDER_ERROR and PROVIDER_STALE — is swallowed and never reaches
the client. Wrapping a provider in a multi-provider silently costs you those
events, with nothing in the API to hint at it.

That is a known gap, open-feature/java-sdk#1882 (gap 1, "child provider event
aggregation and status tracking", High), originally found by hand-comparing
implementations against the js-sdk reference. Reproducing it from the outside,
without knowing it was there, is a fair advertisement for what the TCK is for.

CONFIGURATION_CHANGE is therefore left undeclared, so the scenario is reported as
skipped-with-reason rather than passing on a provider that cannot satisfy it —
the same treatment flagd's STRICT_NUMERIC_TYPING gets. Delete the omission once
#1882 is fixed. Everything else survives delegation unchanged: 25 passed,
4 skipped.

Signed-off-by: Simon Schrottner 
---
 .github/workflows/ci.yml                      |  9 ++-
 tools/provider-tck/README.md                  | 28 ++++++-
 tools/provider-tck/pom.xml                    | 11 +--
 .../providertck/MultiProviderTckTest.java     | 77 +++++++++++++++++++
 4 files changed, 112 insertions(+), 13 deletions(-)
 create mode 100644 tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 769d4216d..de0eaf2e4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,10 +10,11 @@ on:
 
 jobs:
   # Fast canary for the Provider TCK: runs the full applicable conformance suite against the
-  # SDK's InMemoryProvider with no Docker, no Compose stack and no network. It finishes in
-  # seconds, so a broken step definition, a mis-wired capability gate or a regression in the
-  # shared harness is reported long before the containerised provider suites in `main` get
-  # there — and it points at the TCK rather than at whichever provider noticed first.
+  # SDK's InMemoryProvider, and again against MultiProvider wrapping one of them, with no
+  # Docker, no Compose stack and no network. It finishes in seconds, so a broken step
+  # definition, a mis-wired capability gate or a regression in the shared harness is reported
+  # long before the containerised provider suites in `main` get there — and it points at the
+  # TCK rather than at whichever provider noticed first.
   #
   # Deliberately not a gate on `main`: the two run in parallel so a green run is not delayed.
   # The same suite also runs inside `main` as part of the reactor build; this job exists to
diff --git a/tools/provider-tck/README.md b/tools/provider-tck/README.md
index 7370f085f..85e252682 100644
--- a/tools/provider-tck/README.md
+++ b/tools/provider-tck/README.md
@@ -136,10 +136,30 @@ Get that pairing wrong — declare `STALE` against a control that cannot disconn
 `BackendControl` may throw `UnsupportedOperationException` for operations it does not support, and
 reaching one from a scenario that actually ran is a **test-configuration bug**, never a skip.
 
-The TCK's own self-test is exactly this class: see
-[`InMemoryProviderTckTest`](src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java),
-which runs the full applicable suite against `InMemoryProvider` with no Docker in well under a
-second. It doubles as the reference adoption and as the Docker-free CI canary.
+### The TCK's own self-tests
+
+Two suites in this module are exactly the class above, and both run with no Docker in well under a
+second. They are the reference adoption, and they are the fast CI canary.
+
+[`InMemoryProviderTckTest`](src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java)
+runs the full applicable suite against the SDK's `InMemoryProvider` — 26 passed, 3 skipped by
+capability.
+
+[`MultiProviderTckTest`](src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java)
+runs it against `MultiProvider` wrapping **one** `InMemoryProvider`. A provider that delegates is
+still a provider, and delegation is where the contract is easiest to drop: a variant that does not
+survive the hop, a reason rewritten, an error code flattened, an event that never arrives. With a
+single child the correct answer is precisely what the in-memory suite already asserts, so any
+difference between the two suites is attributable to `MultiProvider` and nothing else.
+
+That suite has already paid for itself. It does **not** declare `CONFIGURATION_CHANGE`, because
+`MultiProvider` extends `EventProvider` but never subscribes to its children — a child's
+`PROVIDER_CONFIGURATION_CHANGED`, `PROVIDER_ERROR` and `PROVIDER_STALE` are all swallowed. Wrapping
+a provider in a multi-provider silently costs you those events, with nothing in the API to hint at
+it. That is a known SDK gap,
+[open-feature/java-sdk#1882](https://github.com/open-feature/java-sdk/issues/1882) (gap 1, High),
+which the suite reproduced from the outside — the gap was originally found by hand-comparing
+implementations against the js-sdk reference. Everything else survives delegation unchanged.
 
 ## Adopting it
 
diff --git a/tools/provider-tck/pom.xml b/tools/provider-tck/pom.xml
index ec59f7a08..ee5d7ce9e 100644
--- a/tools/provider-tck/pom.xml
+++ b/tools/provider-tck/pom.xml
@@ -184,11 +184,12 @@
         
 
         
         
             org.junit.jupiter
diff --git a/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java
new file mode 100644
index 000000000..2d34c0094
--- /dev/null
+++ b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java
@@ -0,0 +1,77 @@
+package dev.openfeature.contrib.tools.providertck;
+
+import dev.openfeature.sdk.FeatureProvider;
+import dev.openfeature.sdk.multiprovider.MultiProvider;
+import dev.openfeature.sdk.providers.memory.InMemoryProvider;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Runs the OpenFeature Provider TCK against the SDK's {@link MultiProvider}, wrapping a single
+ * {@link InMemoryProvider}.
+ *
+ * 
A provider that delegates is still a provider, and delegation is where the contract is easiest
+ * to drop on the floor: a variant that does not survive the hop, a reason rewritten to
+ * {@code DEFAULT}, an error code flattened to {@code GENERAL}, an event that never reaches the
+ * client. Wrapping exactly one child makes every one of those observable, because the correct
+ * answer is precisely what {@link InMemoryProviderTckTest} already asserts. Any difference between
+ * these two suites is attributable to {@link MultiProvider} and nothing else.
+ *
+ * 
That framing is the point of running it here rather than in the SDK: this is not a test of
+ * aggregation across several backends, it is a test that delegation is transparent.
+ *
+ * 
It costs one class, needs no Docker, and it has already earned its place — see the capability
+ * note below.
+ */
+public class MultiProviderTckTest extends ProviderTckTest {
+
+    private final InProcessBackendControl control = new InProcessBackendControl();
+
+    @Override
+    public BackendControl backendControl() {
+        return control;
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * 
Exactly one child. See the class javadoc for why that is the interesting configuration
+     * rather than a degenerate one.
+     */
+    @Override
+    public FeatureProvider createProvider() {
+        return new MultiProvider(Collections.singletonList(control.createProvider()));
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * 
{@link Capability#CONFIGURATION_CHANGE} is not declared, and that is a
+     * finding rather than a configuration choice.
+     *
+     * 
{@link MultiProvider} extends {@code EventProvider} but never subscribes to its children,
+     * so a child's {@code PROVIDER_CONFIGURATION_CHANGED} — along with its {@code PROVIDER_ERROR}
+     * and {@code PROVIDER_STALE} — is swallowed and never reaches the client. Wrapping an
+     * in-memory provider in a multi-provider therefore silently costs you configuration-change
+     * events, with nothing in the API to suggest it.
+     *
+     * 
This is a known gap, tracked as
+     * open-feature/java-sdk#1882
+     * (gap 1, "child provider event aggregation and status tracking", High). The suite reproduced
+     * it from the outside, which is a reasonable advertisement for what the TCK is for: the gap was
+     * originally found by hand-comparing implementations against the js-sdk reference.
+     *
+     * 
Delete this omission once #1882 is fixed. Until then the
+     * {@code @configuration-change} scenario is reported as skipped-with-reason rather than passing
+     * on a provider that cannot satisfy it.
+     *
+     * 
Everything else holds. Values, variants, reasons, the full type-mismatch matrix,
+     * {@code FLAG_NOT_FOUND}, structured values, strict numeric typing and reaching {@code READY}
+     * all survive the delegation hop unchanged.
+     */
+    @Override
+    public Set capabilities() {
+        return EnumSet.of(Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING);
+    }
+}