diff --git a/.github/component_owners.yml b/.github/component_owners.yml index 4e4a03415..90a018aa9 100644 --- a/.github/component_owners.yml +++ b/.github/component_owners.yml @@ -46,6 +46,8 @@ components: - toddbaert tools/flagd-http-connector: - liran2000 + tools/provider-tck: + - aepfli ignored-authors: - renovate-bot diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ddd0ff26e..b24f5b13a 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -17,6 +17,7 @@ "tools/flagd-http-connector": "0.0.5", "tools/flagd-api": "1.0.0", "tools/flagd-api-testkit": "0.2.1", + "tools/provider-tck": "0.0.1", "tools/flagd-core": "2.0.1", ".": "1.0.0", "providers/optimizely": "1.0.0" diff --git a/pom.xml b/pom.xml index 29963bb06..ee4835938 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ + tools/provider-tck tools/flagd-api-testkit tools/flagd-api tools/flagd-core diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index fb114bd0c..e759c1a2b 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -22,6 +22,8 @@ 1.2.28 [2.0.0,3.0.0) + + [0.0.1,) flagd @@ -98,6 +100,17 @@ 5.14.3 test + + + dev.openfeature.contrib.tools + provider-tck + ${provider-tck.version} + test + org.testcontainers testcontainers diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java new file mode 100644 index 000000000..4c42307d6 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java @@ -0,0 +1,116 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.providers.flagd.FlagdOptions; +import dev.openfeature.contrib.providers.flagd.FlagdProvider; +import dev.openfeature.contrib.tools.providertck.AbstractProviderTckTest; +import dev.openfeature.contrib.tools.providertck.BackendEndpoint; +import dev.openfeature.contrib.tools.providertck.Capability; +import dev.openfeature.sdk.FeatureProvider; +import java.io.File; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * Shared configuration for running the OpenFeature Provider TCK against the flagd provider. + * + *

flagd resolves flags in two quite different ways, and both are worth conforming: RPC evaluates + * remotely over gRPC, while in-process syncs the ruleset and evaluates locally. They share a backend + * stack and differ only in resolver and port, so the modes are two small subclasses. + * + *

Each concrete subclass is its own JUnit suite and its own TCK harness; the TCK works out which + * one is running from the JUnit test plan, so adding a mode needs no registration or build + * configuration. + */ +abstract class AbstractFlagdTckTest extends AbstractProviderTckTest { + + /** + * A port nothing listens on, for the initialisation-failure scenarios. + * + *

Deliberately not a port on the Compose stack: the stack must stay up for the whole suite, + * and simulated outages belong to the control API. + */ + private static final int UNAVAILABLE_PORT = 9999; + + /** + * gRPC deadline for a provider that is expected to connect. + * + *

Generous on purpose. flagd derives its initialisation deadline from this value, and the + * in-process resolver must sync the entire ruleset before it reports ready — which intermittently + * takes longer than a deadline tuned for a single RPC round trip. + */ + private static final int CONNECTED_DEADLINE_MS = 5000; + + /** + * gRPC deadline for a provider pointed at a dead port. + * + *

Short on purpose, and deliberately not the same as {@link #CONNECTED_DEADLINE_MS}: the + * initialisation-failure scenarios assert that the failure is reported promptly, so a + * provider that takes as long to give up as it does to connect would defeat the point. + */ + private static final int UNAVAILABLE_DEADLINE_MS = 1000; + + /** The resolver under test. */ + protected abstract Config.Resolver resolver(); + + /** The container-internal port that resolver connects to. */ + protected abstract int backendPort(); + + @Override + public File composeFile() { + return new File("src/test/resources/tck/docker-compose.yaml"); + } + + @Override + public List backendPorts() { + return Collections.singletonList(backendPort()); + } + + @Override + public FeatureProvider createProvider(BackendEndpoint endpoint) { + return new FlagdProvider(baseOptions() + .deadline(CONNECTED_DEADLINE_MS) + .host(endpoint.host()) + .port(endpoint.port(backendPort())) + .build()); + } + + @Override + public FeatureProvider createUnavailableProvider() { + return new FlagdProvider(baseOptions() + .deadline(UNAVAILABLE_DEADLINE_MS) + .host("localhost") + .port(UNAVAILABLE_PORT) + .build()); + } + + /** + * {@inheritDoc} + * + *

Everything except {@link Capability#STRICT_NUMERIC_TYPING}. Evaluating {@code float-flag} + * (0.5) through the integer API returns {@code 0} with no error code rather than + * {@code TYPE_MISMATCH} with the code default — the value is silently truncated. That is a + * defect to fix, not a design choice; this override should be deleted once it is. + * + *

Declared here rather than per mode because both resolvers behave identically, which places + * the defect in the shared provider layer rather than in either transport. Every other + * capability, including the full non-numeric type-mismatch matrix, holds in both modes. + * + *

That includes {@link Capability#LIFECYCLE}, and legitimately so: flagd reaches its backend + * during initialisation in both modes — an RPC round trip, or a full ruleset sync — so the + * lifecycle scenarios assert something real here rather than passing vacuously. + */ + @Override + public Set capabilities() { + return EnumSet.complementOf(EnumSet.of(Capability.STRICT_NUMERIC_TYPING)); + } + + private FlagdOptions.FlagdOptionsBuilder baseOptions() { + return FlagdOptions.builder() + .resolverType(resolver()) + .retryGracePeriod(2) + .retryBackoffMs(500); + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/FlagdInProcessTckTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/FlagdInProcessTckTest.java new file mode 100644 index 000000000..1ce5b1dc7 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/FlagdInProcessTckTest.java @@ -0,0 +1,17 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import dev.openfeature.contrib.providers.flagd.Config; + +/** Runs the OpenFeature Provider TCK against the flagd provider in in-process mode. */ +public class FlagdInProcessTckTest extends AbstractFlagdTckTest { + + @Override + protected Config.Resolver resolver() { + return Config.Resolver.IN_PROCESS; + } + + @Override + protected int backendPort() { + return 8015; + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/FlagdRpcTckTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/FlagdRpcTckTest.java new file mode 100644 index 000000000..30ee6db57 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/FlagdRpcTckTest.java @@ -0,0 +1,17 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import dev.openfeature.contrib.providers.flagd.Config; + +/** Runs the OpenFeature Provider TCK against the flagd provider in RPC mode. */ +public class FlagdRpcTckTest extends AbstractFlagdTckTest { + + @Override + protected Config.Resolver resolver() { + return Config.Resolver.RPC; + } + + @Override + protected int backendPort() { + return 8013; + } +} diff --git a/providers/flagd/src/test/resources/tck/docker-compose.yaml b/providers/flagd/src/test/resources/tck/docker-compose.yaml new file mode 100644 index 000000000..4cecaa138 --- /dev/null +++ b/providers/flagd/src/test/resources/tck/docker-compose.yaml @@ -0,0 +1,15 @@ +# Backend stack for the OpenFeature Provider TCK, wrapping the unmodified flagd testbed image. +# +# The image already serves everything the TCK needs: flagd itself, and the "launchpad" control +# API on 8080 whose endpoints this TCK's control API contract was derived from. +# +# Note there are no host port bindings. The TCK requires dynamically mapped ports and discovers +# them after startup — a pinned host port would make the suite unrunnable in parallel and would +# collide with a developer's local flagd. +services: + backend: + image: ghcr.io/open-feature/flagd-testbed:v3.8.0 + ports: + - 8013 # flagd RPC evaluation (gRPC) + - 8015 # flagd in-process sync (gRPC) + - 8080 # launchpad control API diff --git a/release-please-config.json b/release-please-config.json index d3281c2d5..b67946170 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -205,6 +205,17 @@ "README.md" ] }, + "tools/provider-tck": { + "package-name": "dev.openfeature.contrib.tools.providertck", + "release-type": "simple", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "versioning": "default", + "extra-files": [ + "pom.xml", + "README.md" + ] + }, "tools/flagd-api-testkit": { "package-name": "dev.openfeature.contrib.tools.flagdapitestkit", "release-type": "simple", diff --git a/tools/provider-tck/README.md b/tools/provider-tck/README.md new file mode 100644 index 000000000..e9d8232c3 --- /dev/null +++ b/tools/provider-tck/README.md @@ -0,0 +1,361 @@ +# OpenFeature Provider TCK + +A conformance test suite that any OpenFeature provider can adopt to verify it implements the +provider contract of the [OpenFeature specification](https://openfeature.dev/specification/). + +OpenFeature's central promise is that swapping providers does not change application behaviour. +Today nothing verifies that — every provider tests differently, so "implements the provider +contract" is an unverified claim. This is the shared suite that makes it checkable. + +> **Status: proof of concept.** The scenario set is a representative subset covering each +> architectural mechanism once, not exhaustive coverage. See [Known gaps](#known-gaps). + +## Installation + + +```xml + + dev.openfeature.contrib.tools + provider-tck + 0.0.1 + test + +``` + + +Requires Java 11+, JUnit 5, and a working Docker daemon. + +### OpenFeature SDK compatibility + +The TCK declares `dev.openfeature:sdk` as a **`provided` version range** (`[1.21.0,1.99999)`), +inherited from this repository's parent POM. It never pins an SDK version. + +That is deliberate. A conformance suite that forces an SDK upgrade before you can run it is a +conformance suite nobody runs. Your build keeps whatever SDK version it already resolves; the TCK +uses only long-stable API — `OpenFeatureAPI`, `Client`, typed evaluation, `ProviderEvent`, +`ProviderState`. + +## What it tests, and what it does not + +**In scope — the provider contract:** + +- mapping backend responses onto typed resolution details (value, variant, reason, error code) +- keeping the integer and float types distinct +- error handling: type mismatch and unknown flag return the code default, report the right error + code, and never throw +- lifecycle: reaching `READY`, and settling into `ERROR` against an unreachable backend +- events: `PROVIDER_READY`, `PROVIDER_ERROR`, `PROVIDER_STALE`, `PROVIDER_CONFIGURATION_CHANGED` +- that a signalled configuration change is actually applied on re-evaluation + +**Out of scope — not the provider's contract:** + +- backend evaluation logic, targeting and bucketing correctness. Every flag in the canonical set + resolves to its default variant with no targeting, so what is under test is the provider's + mapping of a response, not the backend's decision. +- the provider↔backend wire protocol. How you talk to your backend is your business. +- SDK behaviour. That belongs to the SDK's own test suite. + +## Adopting it + +Four things to implement, then two small files. + +### 1. A Docker Compose stack + +```yaml +# src/test/resources/tck/docker-compose.yaml +services: + backend: + image: your-org/your-testbed:1.0.0 + ports: + - 8080 # control API (see below) + - 5000 # whatever your provider connects to +``` + +Conventions the TCK relies on — all overridable: + +| Convention | Default | Override | +|---|---|---| +| Service hosting the control API and backend | `backend` | `backendService()` | +| Container-internal control API port | `8080` | `controlPort()` | +| Extra services/ports to expose | none | `additionalExposedPorts()` | + +**Never pin host ports.** External ports are mapped dynamically and discovered after startup — +that is why the provider comes from a factory rather than a constant. Pinned ports make the suite +unrunnable in parallel with anything else and collide with a developer's local backend. + +The stack may contain any number of extra containers: a toxiproxy, an edge service, a sidecar. +The TCK only cares about the two conventions above. + +### 2. A control API on the backend + +Your stack must expose a small HTTP control API so the TCK can put the backend into specific +states. The full contract is in [`openapi/control-api.yaml`](src/main/resources/openapi/control-api.yaml), +packaged inside the JAR. Summary: + +| Endpoint | Status | Purpose | +|---|---|---| +| `POST /start?config={name}` | **required** | start the backend, seed flags to that config's baseline | +| `POST /stop` | **required** | make the backend unreachable | +| `POST /restart?seconds={n}` | **required** | bounded outage, flag state preserved | +| `POST /change` | **required** | change `changing-flag`'s resolved value | +| `POST /reset` | optional | restore baseline without an outage; falls back to `/start` | +| `GET /healthz` | optional | readiness; falls back to a TCP port check | + +Two normative requirements are worth repeating here because getting them wrong is subtle: + +> **Never stop or restart a container to simulate an outage.** Testcontainers cannot reliably +> preserve dynamically mapped host ports across a container restart, so a restart silently +> invalidates every provider already pointed at the old port — in some language bindings, and not +> in others, which makes it a portability trap rather than a bug you would catch locally. Simulate +> outages *inside* the running stack: kill the backend process, add a proxy toxic, block the +> socket. The [flagd testbed](https://github.com/open-feature/flagd-testbed) kills and restarts the +> flagd process inside a container that keeps running — that is the reference behaviour. + +> **`/start` resets flag state; `/restart` preserves it.** An outage must be observable as a change +> in availability, never as a change in flag values. The TCK relies on this split for scenario +> isolation. + +### 3. The canonical flag set + +Seed your backend with the flags in [`flags/canonical-flags.json`](src/main/resources/flags/canonical-flags.json). +It is expressed in the flagd flag-definition format because that is the only widely implemented +vendor-neutral format today — the format is not what matters, the keys, types, variants and +resolved values are. Seed them however your backend seeds flags. + +Two details are load-bearing: + +- **`missing-flag` must not exist.** Its absence is what the `FLAG_NOT_FOUND` scenario tests. +- **No flag has targeting rules.** Every scenario expects reason `STATIC`. + +### 4. The test class + +```java +public class MyProviderTckTest extends AbstractProviderTckTest { + + @Override + public File composeFile() { + return new File("src/test/resources/tck/docker-compose.yaml"); + } + + @Override + public List backendPorts() { + return Collections.singletonList(5000); + } + + @Override + public FeatureProvider createProvider(BackendEndpoint endpoint) { + return new MyProvider(endpoint.host(), endpoint.port(5000)); + } + + @Override + public FeatureProvider createUnavailableProvider() { + return new MyProvider("localhost", 9999); + } +} +``` + +That is the whole adoption — one file, no registration. The class is simultaneously the JUnit suite +and the harness, and the TCK works out which suite is running from the JUnit test plan. The Compose +lifecycle, port discovery, control API calls, provider registration, event awaiting and teardown all +belong to the TCK. **If you find yourself adding test infrastructure to this class, that is a bug in +the TCK — please open an issue rather than working around it.** + +`createUnavailableProvider()` should point at a closed port on localhost, not at your stack — the +stack must stay up, and simulated outages belong to the control API. Give it a short connection +deadline; the scenario allows a bounded time for the error event and a 30-second connect timeout +will not make it. + +#### Several provider modes + +A provider with more than one transport writes **one class per mode and nothing else** — no +registration, no system property, no build configuration. Each class is its own suite, each gets its +own Compose stack, and they can share a base class: + +```java +abstract class AbstractMyProviderTckTest extends AbstractProviderTckTest { + protected abstract Mode mode(); + // composeFile(), createProvider(), capabilities() ... shared here +} + +public class MyProviderRemoteTckTest extends AbstractMyProviderTckTest { + @Override protected Mode mode() { return Mode.REMOTE; } +} + +public class MyProviderInProcessTckTest extends AbstractMyProviderTckTest { + @Override protected Mode mode() { return Mode.IN_PROCESS; } +} +``` + +This is how flagd covers RPC and in-process — see +[`AbstractFlagdTckTest`](../../providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java). +Abstract classes are not run, so an intermediate base is safe. + +Note that per-mode differences may include timing, not just wiring: flagd's in-process resolver +syncs the whole ruleset before reporting ready, so it needs a longer initialisation deadline than +its RPC mode. Give a connecting provider a generous deadline and an intentionally unreachable one a +short deadline — the failure scenarios assert that failure is reported *promptly*. + +

+Fallback: ServiceLoader registration + +Suite discovery relies on the JUnit Platform auto-registering `TckSuiteListener` (declared in this +JAR's `META-INF/services/org.junit.platform.launcher.TestExecutionListener`), which Surefire, Gradle +and IDEs all do by default. If your launcher disables listener auto-registration, register the +harness explicitly instead at +`src/test/resources/META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness`, +and if you register more than one, select between them with +`-Dopenfeature.tck.harness=MyProviderRemoteTckTest`. + +
+ +## Declaring capabilities + +Not every provider implements every optional part of the spec. Scenarios that exercise an optional +capability carry a tag; declare which ones you support and the rest are reported as **skipped**, +with the reason printed. They are never silently passed — a conformance suite that quietly goes +green on scenarios it did not run is worse than no suite at all. + +| Capability | Tag | Meaning | +|---|---|---| +| `LIFECYCLE` | `@lifecycle` | performs an initialisation that reaches its backend, with an observable outcome | +| `EVENTS` | `@events` | emits lifecycle events at all | +| `STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | +| `CONFIGURATION_CHANGE` | `@configuration-change` | detects config changes, emits `PROVIDER_CONFIGURATION_CHANGED` | +| `OBJECT` | `@object` | supports structured flag values | +| `UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging on a dead backend | +| `STRICT_NUMERIC_TYPING` | `@strict-numeric-typing` | does not coerce between integer and float | +| `TARGETING` | `@targeting` | reserved, no scenarios yet | +| `CACHING` | `@caching` | reserved, no scenarios yet | + +The default is every capability. **Narrow it, do not widen it**: start from the default, run the +suite, and remove only what your provider genuinely cannot do. + +```java +@Override +public Set capabilities() { + return EnumSet.complementOf(EnumSet.of(Capability.STALE, Capability.CACHING)); +} +``` + +A note on `LIFECYCLE` vs `EVENTS`: they look like the same thing and are not. `EVENTS` says the +provider emits events; `LIFECYCLE` says there is a real initialisation behind them. The SDK's +`FeatureProviderStateManager` emits `PROVIDER_READY`/`PROVIDER_ERROR` around `initialize` for *any* +provider, `EventProvider` or not — so a provider that does no initialisation of its own reaches +`READY` exactly as `NoOpProvider` would, and gating the readiness scenario on `EVENTS` would pass it +vacuously. Conversely a stateless provider such as OFREP genuinely initialises against a backend +while emitting no events of its own, and would have been excluded. Declare `LIFECYCLE` only if +initialisation actually talks to the backend; a provider with nothing to reach — an in-memory +provider, or a facade over other providers — should not declare it however many events it emits. + +A note on `STRICT_NUMERIC_TYPING`: unlike the others it is not an optional feature. The spec +requires `TYPE_MISMATCH` when the requested type cannot be satisfied, and narrowing `0.5` to `0` +loses information silently — the worst failure mode for a feature flag, because the application +sees a plausible value and no error. It is a capability only so a provider with this defect can +adopt the TCK today and see the gap reported explicitly. Not declaring it is an admission of a +known bug. **The flagd provider currently does not declare it**, in either RPC or in-process mode — +see [`AbstractFlagdTckTest`](../../providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java). + +## Tuning timeouts + +How fast a provider notices a backend change differs by orders of magnitude between transports: a +streaming provider sees a configuration change in milliseconds, a provider polling every 30 seconds +needs most of a poll interval. Every await timeout is therefore overridable. + +| Method | Default | What it bounds | +|---|---|---| +| `eventTimeout()` | 12s | waiting for a provider event | +| `readyTimeout()` | 30s | waiting for a provider to reach a lifecycle state | +| `startupTimeout()` | 60s | bringing the Compose stack up | +| `settleTime()` | 50ms | pause after a control API call | + +```java +@Override +public Duration eventTimeout() { + return Duration.ofSeconds(45); // we poll every 30s +} +``` + +Set `eventTimeout()` to comfortably exceed your worst-case detection latency, or the suite reports +timeouts that are really just impatience. Scenarios that assert promptness as part of their point +use the explicit `within {int}ms` step, which always wins. + +## Running it + +```bash +mvn test -Dtest=MyProviderTckTest +``` + +Scenarios run **serially** and the suite enforces this, overriding any +`cucumber.execution.parallel.enabled=true` in your module's `junit-platform.properties`. Control API +state is global to the Compose stack, so concurrent scenarios corrupt each other — one scenario's +`/start` restarts the backend underneath another's disconnect assertion. The symptom looks like a +flaky provider rather than a broken test, which is exactly why it is enforced rather than +documented. + +The Compose stack starts once per suite and is never restarted. Scenario isolation comes from the +control API. + +## Relationship to the flagd test harness + +The step vocabulary is inherited from the +[flagd test harness](https://github.com/open-feature/test-harness) wherever it was already +provider-neutral, so flagd's existing feature files port with a near-zero diff and the step +definitions stay familiar. Only genuinely flagd-specific wording was renamed: + +| flagd test harness | Provider TCK | Why | +|---|---|---| +| `Given a stable flagd provider` | `Given a stable provider` | drops the vendor name | +| `Given a unavailable flagd provider` | `Given a unavailable provider` | drops the vendor name | + +Everything else is unchanged: `a -flag with key ... and a default value ...`, +`the flag was evaluated with details`, `the resolved details value should be "..."`, +`the reason should be ...`, `the variant should be ...`, `the error-code should be ...`, +`a event handler`, `the event handler should have been executed[ within ms]`, +`the connection is lost[ for s]`, `the flag was modified`, +`the flag should be part of the event payload`, `the client should be in state`. + +Three steps are new: + +| Step | Why it was added | +|---|---| +| `When the connection is restored` | the flagd harness only has the self-healing `lost for {int}s` form, which cannot express "assert stale, *then* reconnect" — the reconnect races the assertion | +| `When the resolved value is remembered` / `Then the resolved details value should have changed` | the control API only requires that `/change` changes `changing-flag`'s value, not which value it changes to; asserting a delta keeps the scenario vendor-neutral | +| `Then no exception should have been thrown` | makes the "never throws" half of the error contract explicit rather than implicit in a step failure | + +## Where these artifacts should live + +The feature files, the control API spec and the canonical flag set are **not Java artifacts**. They +are language-agnostic definitions of the provider contract that every language's TCK must agree on +byte for byte, and that backend vendors implement in whatever language their testbed is written in. + +They belong in the OpenFeature [spec repository](https://github.com/open-feature/spec), with this +module as their Java delivery vehicle. The three travel together by necessity: a feature file that +evaluates `boolean-flag` is meaningless without the flag definition, and a disconnect scenario is +meaningless without the endpoint that produces the disconnect. + +They live here for now only because the PoC had to start somewhere. Moving them changes nothing for +consumers — the features stay on the classpath and stay inside the JAR. + +## Known gaps + +- **Evaluation context passthrough.** The TCK builds evaluation contexts but cannot assert the + context *reached* the backend intact. That needs an echo operation on the control API — something + like `GET /last-evaluation` returning the request the backend last received. Until then, a + provider that silently drops the context passes. +- **Targeting and bucketing.** Out of scope by design: that is backend evaluation logic. The + `@targeting` tag is reserved for context-passthrough scenarios once the gap above is closed. +- **Caching.** Whether a stale provider keeps serving last-known values during an outage depends on + whether it holds a local copy of the ruleset. The `@caching` tag is reserved; no scenarios yet. +- **Hooks.** Not covered. +- **Flag metadata.** The flagd harness has metadata scenarios; they are not yet ported. +- **Multi-suite JVMs.** `TckRuntime` is static, so TCK suites run one at a time within a JVM fork. + Several suites in one fork is fine — they run sequentially, each with its own Compose stack — but + they cannot run concurrently. +- **Scenario coverage is a representative subset**, covering each architectural mechanism once + rather than exhaustively. + +## Contributing + +See the repository [CONTRIBUTING.md](../../CONTRIBUTING.md). New scenarios should be portable +across providers: if a scenario can only pass against one vendor's backend semantics, it belongs in +that provider's own suite, not here. diff --git a/tools/provider-tck/lombok.config b/tools/provider-tck/lombok.config new file mode 100644 index 000000000..df71bb6a0 --- /dev/null +++ b/tools/provider-tck/lombok.config @@ -0,0 +1,2 @@ +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/tools/provider-tck/pom.xml b/tools/provider-tck/pom.xml new file mode 100644 index 000000000..3de834808 --- /dev/null +++ b/tools/provider-tck/pom.xml @@ -0,0 +1,185 @@ + + + 4.0.0 + + dev.openfeature.contrib + parent + [1.0,2.0) + ../../pom.xml + + dev.openfeature.contrib.tools + provider-tck + 0.0.1 + + + ${groupId}.providertck + 3.27.7 + 4.3.0 + 2.22.1 + 2.0.17 + 2.0.4 + 1.3.0 + + + provider-tck + + Language-agnostic conformance test suite (TCK) for OpenFeature providers. + Bundles the canonical Gherkin feature files, Cucumber step definitions and an + abstract JUnit Platform Suite base class that owns the full test lifecycle: + starting the vendor-supplied Docker Compose stack, discovering dynamically + mapped ports, driving the standardised control API and awaiting provider + events. Provider authors implement a single factory interface. + + https://openfeature.dev + + + + aepfli + Simon Schrottner + OpenFeature + https://openfeature.dev/ + + + + + + + + + + + io.cucumber + cucumber-java + + + + + + io.cucumber + cucumber-junit-platform-engine + + + + + io.cucumber + cucumber-picocontainer + compile + + + + + org.junit.platform + junit-platform-suite + compile + + + + + org.junit.platform + junit-platform-launcher + compile + + + + + org.opentest4j + opentest4j + ${opentest4j.version} + compile + + + + + org.assertj + assertj-core + ${assertj.version} + compile + + + + + org.awaitility + awaitility + ${awaitility.version} + compile + + + + + org.testcontainers + testcontainers + ${testcontainers.version} + compile + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java new file mode 100644 index 000000000..4f3304d94 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java @@ -0,0 +1,46 @@ +package dev.openfeature.contrib.tools.providertck; + +import io.cucumber.junit.platform.engine.Constants; +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +/** + * Base JUnit Platform Suite for the OpenFeature Provider TCK. + * + *

Carries all Cucumber runner configuration so that a provider author writes no test + * infrastructure at all. The canonical feature files are packaged inside this JAR and selected from + * the classpath, so consumers need no git submodule of their own. + * + *

To adopt the TCK, extend this class, implement the four abstract methods of + * {@link ProviderTckHarness}, and register the concrete class in + * {@code src/test/resources/META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness}. + * + *

Scenarios run serially, and this class enforces that rather than merely + * asking for it. Control API state — which flags are seeded, whether the backend is reachable — is + * global to the Compose stack, so concurrent scenarios corrupt each other: one scenario's + * {@code /start} restarts the backend underneath another's disconnect assertion. The failure looks + * like a flaky provider rather than a broken test, which makes it expensive to diagnose. + * + *

The suite therefore pins {@code cucumber.execution.parallel.enabled=false} here, where it + * overrides any {@code junit-platform.properties} the consuming module happens to ship. Several + * providers already enable Cucumber parallelism for their own suites, and inheriting that setting + * silently breaks the TCK. + * + *

Note this class carries no lifecycle code. The Compose stack, the control API client, provider + * registration and event awaiting are all owned by the step definitions in + * {@code dev.openfeature.contrib.tools.providertck.steps}, which reach the harness through + * {@link TckRuntime}. + * + * @see ProviderTckHarness + */ +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource("features") +@ConfigurationParameter(key = Constants.PLUGIN_PROPERTY_NAME, value = "summary") +@ConfigurationParameter(key = Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = "false") +@ConfigurationParameter(key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, value = "same_thread") +@ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.tools.providertck.steps") +@ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") +public abstract class AbstractProviderTckTest implements ProviderTckHarness {} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java new file mode 100644 index 000000000..40f7799cf --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java @@ -0,0 +1,77 @@ +package dev.openfeature.contrib.tools.providertck; + +import org.testcontainers.containers.ComposeContainer; + +/** + * Addresses of the running backend stack, handed to + * {@link ProviderTckHarness#createProvider(BackendEndpoint)}. + * + *

This type exists because external ports are only known after the Compose stack has + * started. Compose stacks under test must not pin host ports — Docker assigns them dynamically, so + * a provider cannot be configured until the stack is up. That is the whole reason the harness + * exposes a factory method rather than a pre-built provider instance. + * + *

The port mapping is stable for the lifetime of the suite: the stack is started once and never + * restarted, so a provider built from this endpoint stays valid across every scenario. See the + * no-container-restart invariant in {@code openapi/control-api.yaml}. + */ +public final class BackendEndpoint { + + private final ComposeContainer compose; + private final String defaultService; + + BackendEndpoint(ComposeContainer compose, String defaultService) { + this.compose = compose; + this.defaultService = defaultService; + } + + /** + * Returns the host the stack is reachable on. + * + *

This is not necessarily {@code localhost}: with a remote Docker daemon, Docker Desktop on + * some platforms, or a rootless setup, it can be an arbitrary address. Always use this value + * rather than hard-coding a host. + * + * @return the Docker host serving the backend stack + */ + public String host() { + return compose.getServiceHost(defaultService, null); + } + + /** + * Returns the host the named service is reachable on. + * + * @param service the Compose service name + * @return the Docker host serving that service + */ + public String host(String service) { + return compose.getServiceHost(service, null); + } + + /** + * Resolves the dynamically mapped host port for a container-internal port on the default + * backend service. + * + * @param internalPort the container-internal port, as declared by + * {@link ProviderTckHarness#backendPorts()} + * @return the host port the service is reachable on + */ + public int port(int internalPort) { + return port(defaultService, internalPort); + } + + /** + * Resolves the dynamically mapped host port for a container-internal port on a named service. + * + *

Use this for multi-service stacks — a proxy, an edge service, a sidecar. The service and + * port must have been declared via {@link ProviderTckHarness#additionalExposedPorts()}, + * otherwise Testcontainers has not exposed it and this call fails. + * + * @param service the Compose service name + * @param internalPort the container-internal port + * @return the host port the service is reachable on + */ + public int port(String service, int internalPort) { + return compose.getServicePort(service, internalPort); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java new file mode 100644 index 000000000..38bcb5898 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java @@ -0,0 +1,121 @@ +package dev.openfeature.contrib.tools.providertck; + +import java.util.Arrays; +import java.util.Optional; + +/** + * An optional part of the OpenFeature provider contract that a provider may or may not support. + * + *

Not every provider implements every spec feature — a provider backed by a static file has no + * meaningful notion of going stale, and a provider without a streaming transport cannot emit + * configuration-change events. Rather than forcing such providers to fail scenarios they were never + * going to satisfy, the TCK lets each one declare what it supports via + * {@link ProviderTckHarness#capabilities()}. + * + *

Every capability corresponds to exactly one Gherkin tag. Scenarios carrying a tag whose + * capability was not declared are aborted before they run and are reported as skipped — + * never as passed. Silently green scenarios would make a conformance suite worthless. + * + *

Scenarios with no capability tag are considered mandatory and always run. + */ +public enum Capability { + + /** + * Provider performs an initialisation that reaches its backend, with an observable outcome. + * + *

Gates the lifecycle scenarios: reaching {@code READY} against a healthy backend, and + * settling into {@code ERROR} — promptly, rather than blocking forever or throwing out of + * provider registration — against one that cannot be reached. + * + *

Why this is not {@link #EVENTS}. Gating these scenarios on {@code EVENTS} + * is wrong in both directions. Too strict, because a stateless provider that emits no events of + * its own — OFREP, for instance — still initialises against a backend and still owes the + * contract; it simply cannot declare {@code EVENTS}. Too lax, because a provider that declares + * {@code EVENTS} passes the readiness scenario vacuously: + * {@code dev.openfeature.sdk.FeatureProviderStateManager} emits {@code PROVIDER_READY} and + * {@code PROVIDER_ERROR} around {@code initialize} for any provider, whether or not it + * is an {@code EventProvider}. A provider with no initialisation of its own therefore reaches + * {@code READY} exactly as {@code NoOpProvider} would, and the scenario goes green having + * demonstrated nothing about the provider. + * + *

So {@code EVENTS} asserts that the provider emits events; {@code LIFECYCLE} asserts that + * there is a real initialisation behind the event whose outcome the events describe. Declare it + * only if initialisation actually talks to the backend. A provider with nothing to reach — one + * backed by an in-memory map, or a facade over other providers — should not + * declare it, however many events it emits. + */ + LIFECYCLE("@lifecycle"), + + /** Provider emits lifecycle events at all ({@code PROVIDER_READY}, {@code PROVIDER_ERROR}). */ + EVENTS("@events"), + + /** Provider enters {@code STALE} and emits {@code PROVIDER_STALE} when the backend is lost. */ + STALE("@stale"), + + /** Provider detects flag configuration changes and emits {@code PROVIDER_CONFIGURATION_CHANGED}. */ + CONFIGURATION_CHANGE("@configuration-change"), + + /** Provider supports structured (object) flag values. */ + OBJECT("@object"), + + /** Provider reports an error state rather than hanging when initialised against a dead backend. */ + UNAVAILABLE_INIT("@unavailable"), + + /** + * Provider keeps the integer and float types distinct instead of coercing between them. + * + *

Unlike the other entries here this is not an optional spec feature. The + * specification requires a provider to report {@code TYPE_MISMATCH} when the requested type + * cannot be satisfied, and narrowing {@code 0.5} to {@code 0} to satisfy an integer request + * loses information silently — the worst possible failure mode for a feature flag, because the + * application sees a plausible value and no error. + * + *

It is a capability only so that a provider with this defect can adopt the TCK today and + * see the gap reported as an explicit skip, rather than being unable to adopt at all. Not + * declaring it is an admission of a known bug, not a design choice. Declare it as soon as the + * provider is fixed. + */ + STRICT_NUMERIC_TYPING("@strict-numeric-typing"), + + /** + * Provider supports targeting rules driven by evaluation context. + * + *

Reserved. No scenario in the current suite carries this tag — targeting is backend + * evaluation logic, which the TCK deliberately does not test. It exists so the tag vocabulary + * stays aligned with the flagd test harness and so context-passthrough scenarios have a home + * once the control API grows an echo endpoint. + */ + TARGETING("@targeting"), + + /** + * Provider caches evaluation results and invalidates them on configuration change. + * + *

Reserved; no scenario carries this tag yet. + */ + CACHING("@caching"); + + private final String tag; + + Capability(String tag) { + this.tag = tag; + } + + /** + * Returns the Gherkin tag, including the leading {@code @}, that gates this capability. + * + * @return the Gherkin tag for this capability + */ + public String tag() { + return tag; + } + + /** + * Looks up the capability gated by a Gherkin tag. + * + * @param tag a Gherkin tag including the leading {@code @} + * @return the matching capability, or empty if the tag does not gate a capability + */ + public static Optional fromTag(String tag) { + return Arrays.stream(values()).filter(c -> c.tag.equals(tag)).findFirst(); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java new file mode 100644 index 000000000..6c0a19e57 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java @@ -0,0 +1,228 @@ +package dev.openfeature.contrib.tools.providertck; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for the standardised backend control API. + * + *

Implements the contract in {@code openapi/control-api.yaml}, including the documented fallback + * for the optional {@code /reset} operation. Uses the JDK HTTP client so that adopting the TCK does + * not drag an HTTP library onto a provider's test classpath. + * + *

Every operation here manipulates the backend process or its flag state. None of them + * touch containers — that is the no-container-restart invariant, and it is the reason a provider + * built once at suite start stays valid for every scenario. + */ +public final class ControlApiClient { + + private static final Logger log = LoggerFactory.getLogger(ControlApiClient.class); + + private final HttpClient http; + private final String baseUrl; + private final Duration settleTime; + + /** + * Tri-state cache of whether the backend implements the optional {@code /reset} operation. + * {@code null} until the first {@link #reset(String)} call probes it. + */ + private Boolean resetSupported; + + /** + * Whether the backend was last known to be unreachable. Conservative: {@code restart} sets it + * even though the backend comes back on its own, because a scenario may end before it does. + */ + private boolean backendStopped; + + ControlApiClient(String baseUrl, Duration settleTime) { + this.baseUrl = baseUrl; + this.settleTime = settleTime; + this.http = + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + } + + /** + * Returns the base URL of the control API, for diagnostics. + * + * @return the control API base URL + */ + public String baseUrl() { + return baseUrl; + } + + /** + * Starts the backend with a named configuration, seeding flag state to that configuration's + * baseline. + * + * @param config the configuration name + */ + public void start(String config) { + post("/start?config=" + config); + backendStopped = false; + } + + /** + * Makes the backend unreachable without stopping its container. + */ + public void stop() { + post("/stop"); + backendStopped = true; + } + + /** + * Makes the backend unreachable for a bounded duration, then starts it again. + * + *

Flag state is preserved across the outage, so a provider observes an availability change + * and not a configuration change. + * + * @param seconds how long the backend stays unreachable + */ + public void restart(int seconds) { + post("/restart?seconds=" + seconds); + backendStopped = true; + } + + /** + * Puts the backend into the state every scenario starts from: running, with flag state at the + * baseline of the default configuration. + * + *

Prefers {@link #reset(String)} when the backend is already running, because restoring the + * baseline without an availability blip means the previous scenario's teardown cannot leak a + * spurious lifecycle event into the next scenario. When the previous scenario left the backend + * unreachable, {@code /reset} alone would not bring it back, so this falls through to + * {@link #start(String)}. + * + * @param defaultConfig the configuration name defining the baseline + */ + public void prepareScenario(String defaultConfig) { + if (backendStopped) { + start(defaultConfig); + } else { + reset(defaultConfig); + } + } + + /** + * Mutates flag configuration so that a conforming provider observes a configuration change and + * resolves a different value for {@code changing-flag} afterwards. + */ + public void change() { + post("/change"); + } + + /** + * Restores flag state to the seeded baseline for scenario isolation. + * + *

Prefers the optional {@code POST /reset}, which causes no availability blip. When the + * backend answers {@code 404} or {@code 501} the result is cached and every subsequent call + * falls back to {@code POST /start?config=...}, which resets state at the cost of a process + * restart. Both paths are conformant; see {@code openapi/control-api.yaml}. + * + * @param defaultConfig the configuration name to fall back to + */ + public void reset(String defaultConfig) { + if (Boolean.FALSE.equals(resetSupported)) { + start(defaultConfig); + return; + } + HttpResponse response = send("/reset"); + if (response.statusCode() == 404 || response.statusCode() == 501) { + if (resetSupported == null) { + log.info( + "Control API at {} does not implement POST /reset (HTTP {}); " + + "falling back to POST /start?config={} for scenario isolation.", + baseUrl, + response.statusCode(), + defaultConfig); + } + resetSupported = false; + start(defaultConfig); + return; + } + expectSuccess("/reset", response); + resetSupported = true; + settle(); + } + + /** + * Waits until the control API accepts commands. + * + *

Probes the optional {@code GET /healthz}. A {@code 404} is a conformant answer meaning "not + * implemented", in which case readiness has already been established by the Testcontainers + * listening-port wait strategy and this returns immediately. + * + * @param timeout how long to keep probing + */ + public void awaitReady(Duration timeout) { + long deadline = System.nanoTime() + timeout.toNanos(); + RuntimeException last = null; + while (System.nanoTime() < deadline) { + try { + HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create(baseUrl + "/healthz")) + .GET() + .timeout(Duration.ofSeconds(5)) + .build(), + HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() == 200 || response.statusCode() == 404) { + return; + } + last = new IllegalStateException("control API not ready, HTTP " + response.statusCode()); + } catch (IOException e) { + last = new IllegalStateException("control API not reachable at " + baseUrl, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the control API", e); + } + sleep(Duration.ofMillis(200)); + } + throw new IllegalStateException("control API at " + baseUrl + " did not become ready within " + timeout, last); + } + + private void post(String path) { + expectSuccess(path, send(path)); + settle(); + } + + private HttpResponse send(String path) { + HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + path)) + .POST(HttpRequest.BodyPublishers.noBody()) + .timeout(Duration.ofSeconds(30)) + .build(); + try { + return http.send(request, HttpResponse.BodyHandlers.discarding()); + } catch (IOException e) { + throw new IllegalStateException("control API call POST " + baseUrl + path + " failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted during control API call POST " + path, e); + } + } + + private void expectSuccess(String path, HttpResponse response) { + if (response.statusCode() != 200) { + throw new IllegalStateException( + "control API call POST " + baseUrl + path + " returned HTTP " + response.statusCode() + + ", expected 200. See openapi/control-api.yaml for the expected contract."); + } + } + + private void settle() { + sleep(settleTime); + } + + private static void sleep(Duration duration) { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the backend to settle", e); + } + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/FlagUnderTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/FlagUnderTest.java new file mode 100644 index 000000000..b339dc282 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/FlagUnderTest.java @@ -0,0 +1,57 @@ +package dev.openfeature.contrib.tools.providertck; + +/** + * The flag a scenario is currently exercising: its key, its declared type, and the code default + * passed to the evaluation call. + * + *

The declared type is what makes the integer/float distinction testable. The TCK dispatches to + * {@code getIntegerDetails} or {@code getDoubleDetails} purely on this value, so a provider that + * silently widens an integer to a double is caught rather than accommodated. + */ +public final class FlagUnderTest { + + private final String key; + private final String type; + private final Object defaultValue; + + /** + * Creates a flag under test. + * + * @param key the flag key + * @param type the declared type, one of {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @param defaultValue the code default passed to the evaluation call + */ + public FlagUnderTest(String key, String type, Object defaultValue) { + this.key = key; + this.type = type; + this.defaultValue = defaultValue; + } + + /** + * Returns the flag key. + * + * @return the flag key + */ + public String key() { + return key; + } + + /** + * Returns the declared flag type. + * + * @return the declared flag type + */ + public String type() { + return type; + } + + /** + * Returns the code default passed to the evaluation call. + * + * @return the code default + */ + public Object defaultValue() { + return defaultValue; + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderEventRecord.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderEventRecord.java new file mode 100644 index 000000000..20b0c6957 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderEventRecord.java @@ -0,0 +1,47 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.EventDetails; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * A provider event observed by a scenario's event handler, tagged with the Gherkin word that + * registered the handler ({@code ready}, {@code error}, {@code stale}, {@code change}). + */ +@SuppressFBWarnings( + value = "EI_EXPOSE_REP", + justification = "The SDK's event payload is held and handed on as-is; copying it would " + + "hide exactly the object under assertion") +public final class ProviderEventRecord { + + private final String type; + private final EventDetails details; + + /** + * Records an observed event. + * + * @param type the Gherkin event word the handler was registered under + * @param details the event payload delivered by the SDK + */ + public ProviderEventRecord(String type, EventDetails details) { + this.type = type; + this.details = details; + } + + /** + * Returns the Gherkin event word. + * + * @return the event type word + */ + public String type() { + return type; + } + + /** + * Returns the event payload delivered by the SDK. + * + * @return the event details + */ + public EventDetails details() { + return details; + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java new file mode 100644 index 000000000..1cc671faf --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java @@ -0,0 +1,217 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.FeatureProvider; +import java.io.File; +import java.time.Duration; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * The complete contract a provider author implements to run the OpenFeature Provider TCK. + * + *

Four methods have no default and must be supplied. Everything else is a convention with a + * working default. If you find yourself needing to add lifecycle code, container handling or event + * plumbing to your implementation, that is a bug in the TCK's base class rather than something to + * work around here. + * + *

Implementations are discovered through {@link java.util.ServiceLoader}. Extend + * {@link AbstractProviderTckTest} — which implements this interface and carries all the Cucumber + * configuration — and register the concrete class in + * {@code META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness}. + * + *

Example — the entire adoption for a provider: + * + * 

{@code
+ * public class MyProviderTckTest extends AbstractProviderTckTest {
+ *
+ *     @Override
+ *     public File composeFile() {
+ *         return new File("src/test/resources/tck/docker-compose.yaml");
+ *     }
+ *
+ *     @Override
+ *     public List backendPorts() {
+ *         return Collections.singletonList(8013);
+ *     }
+ *
+ *     @Override
+ *     public FeatureProvider createProvider(BackendEndpoint endpoint) {
+ *         return new MyProvider(endpoint.host(), endpoint.port(8013));
+ *     }
+ *
+ *     @Override
+ *     public FeatureProvider createUnavailableProvider() {
+ *         return new MyProvider("localhost", 9999);
+ *     }
+ * }
+ * }
+ */ +public interface ProviderTckHarness { + + /** + * Returns the Docker Compose file describing the backend stack under test. + * + *

The path is resolved relative to the Maven module directory, so + * {@code new File("src/test/resources/tck/docker-compose.yaml")} is the idiomatic form. + * + *

The stack is started once before the first scenario and stopped after the last one. It is + * never restarted in between — see {@link #createProvider(BackendEndpoint)} and the + * no-container-restart invariant documented in {@code openapi/control-api.yaml}. The stack must + * not pin host ports. + * + * @return the Compose file describing the backend stack + */ + File composeFile(); + + /** + * Returns the container-internal ports on {@link #backendService()} that the provider connects + * to, so Testcontainers can expose and map them. + * + *

The control API port from {@link #controlPort()} is exposed automatically and does not + * need to be listed here. + * + * @return container-internal ports the provider connects to + */ + List backendPorts(); + + /** + * Creates the provider under test, configured against the running backend. + * + *

Called after the Compose stack is up and the control API has seeded the canonical flag + * set. The endpoint carries the dynamically mapped host ports, which is why this is a factory + * rather than a field: the ports do not exist until the stack has started. + * + *

The TCK owns the provider lifecycle from here — it registers the provider with the + * OpenFeature API under a scenario-scoped domain, waits for it to become ready, and shuts it + * down afterwards. Do not call {@code setProvider} or {@code initialize} yourself. + * + * @param endpoint host and mapped ports of the running backend stack + * @return a configured, uninitialised provider + */ + FeatureProvider createProvider(BackendEndpoint endpoint); + + /** + * Creates a provider pointed at a backend that does not exist. + * + *

Used by the initialisation-failure scenarios, which assert that a provider that cannot + * reach its backend settles into {@code ERROR} and emits {@code PROVIDER_ERROR} rather than + * hanging or throwing out of {@code setProvider}. + * + *

Point this at a closed port on localhost. Do not point it at the Compose stack — the stack + * must stay up and reachable, and simulated outages belong to the control API. + * + *

Configure a short connection deadline. The scenario allows a bounded time for the error + * event to arrive, and a provider with a 30-second connect timeout will not make it. + * + * @return a configured provider that cannot reach a backend + */ + FeatureProvider createUnavailableProvider(); + + /** + * Declares which optional parts of the provider contract this provider supports. + * + *

Scenarios tagged with a capability that is not in this set are reported as + * skipped. They are never silently passed. + * + *

Defaults to every capability. Narrow it rather than widening it: start from the default, + * run the suite, and remove only what your provider genuinely cannot do. + * + * @return the capabilities this provider supports + */ + default Set capabilities() { + return EnumSet.allOf(Capability.class); + } + + /** + * Returns the Compose service name that hosts the control API and the backend the provider + * connects to. + * + * @return the Compose service name, {@code backend} by default + */ + default String backendService() { + return "backend"; + } + + /** + * Returns the container-internal port the control API listens on. + * + * @return the control API port, {@code 8080} by default + */ + default int controlPort() { + return 8080; + } + + /** + * Returns extra services and container-internal ports to expose, for stacks that contain more + * than the backend service. + * + *

Keys are Compose service names, values are container-internal ports. Resolve the mapped + * ports with {@link BackendEndpoint#port(String, int)}. + * + * @return additional services and ports to expose, empty by default + */ + default Map> additionalExposedPorts() { + return Collections.emptyMap(); + } + + /** + * Returns the control API configuration name used to seed the canonical flag set. + * + * @return the configuration name passed to {@code POST /start}, {@code default} by default + */ + default String defaultConfig() { + return "default"; + } + + /** + * Returns how long to wait for the Compose stack to become reachable. + * + * @return the stack startup timeout, 60 seconds by default + */ + default Duration startupTimeout() { + return Duration.ofSeconds(60); + } + + /** + * Returns how long to wait for a provider event to arrive. + * + *

This is the single most important knob for a provider author, because providers observe + * backend changes on wildly different timescales. A streaming provider sees a configuration + * change in milliseconds; a provider that polls every 30 seconds may need most of a poll + * interval before it notices. Set this to comfortably exceed your worst-case detection latency, + * or the suite will report timeouts that are really just impatience. + * + *

Individual scenarios can tighten this with the explicit + * {@code within {int}ms} step, which always wins over this value. + * + * @return the default event await timeout, 12 seconds by default + */ + default Duration eventTimeout() { + return Duration.ofSeconds(12); + } + + /** + * Returns how long to wait for a provider to reach {@code READY} during initialisation. + * + * @return the readiness timeout, 30 seconds by default + */ + default Duration readyTimeout() { + return Duration.ofSeconds(30); + } + + /** + * Returns how long to pause after a control API call before continuing. + * + *

Covers the gap between the control API acknowledging a command and the backend actually + * having acted on it. Raise it if you see flakiness immediately after + * {@code the flag was modified} or a provider setup step. + * + * @return the settle time, 50 milliseconds by default + */ + default Duration settleTime() { + return Duration.ofMillis(50); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java new file mode 100644 index 000000000..82da5aa11 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java @@ -0,0 +1,204 @@ +package dev.openfeature.contrib.tools.providertck; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +/** + * Suite-scoped runtime: discovers the provider's harness, owns the Compose stack, and exposes the + * control API client to the step definitions. + * + *

The Compose stack is started once, before the first scenario, and stopped + * after the last one. It is never stopped or restarted in between. Testcontainers cannot reliably + * preserve dynamically mapped host ports across a container restart, so a restart would silently + * invalidate every provider already pointed at the old port. Backend unavailability is therefore + * always simulated inside the running stack through the control API. See the normative statement of + * this invariant in {@code openapi/control-api.yaml}. + * + *

State is static because Cucumber's {@code @BeforeAll} / {@code @AfterAll} hooks are static and + * the stack must outlive individual scenarios. Consequently only one TCK suite may run per JVM fork + * at a time. + */ +@SuppressFBWarnings( + value = "EI_EXPOSE_REP", + justification = "The harness and control API client are shared collaborators by design; " + + "step definitions must act on the same instances the suite started") +public final class TckRuntime { + + private static final Logger log = LoggerFactory.getLogger(TckRuntime.class); + + /** System property selecting a harness by simple class name when several are registered. */ + public static final String HARNESS_SELECTOR_PROPERTY = "openfeature.tck.harness"; + + private static TckRuntime instance; + + private final ProviderTckHarness harness; + private final ComposeContainer compose; + private final ControlApiClient controlApi; + private final BackendEndpoint endpoint; + + private TckRuntime(ProviderTckHarness harness, ComposeContainer compose) { + this.harness = harness; + this.compose = compose; + this.endpoint = new BackendEndpoint(compose, harness.backendService()); + String baseUrl = "http://" + compose.getServiceHost(harness.backendService(), null) + ":" + + compose.getServicePort(harness.backendService(), harness.controlPort()); + this.controlApi = new ControlApiClient(baseUrl, harness.settleTime()); + } + + /** + * Starts the Compose stack if it is not already running, and returns the shared runtime. + * + * @return the suite-scoped runtime + */ + public static synchronized TckRuntime startIfNeeded() { + if (instance == null) { + ProviderTckHarness harness = discoverHarness(); + log.info("Provider TCK harness: {}", harness.getClass().getName()); + instance = new TckRuntime(harness, startCompose(harness)); + instance.controlApi.awaitReady(harness.startupTimeout()); + log.info("Control API ready at {}", instance.controlApi.baseUrl()); + } + return instance; + } + + /** + * Stops the Compose stack and releases the shared runtime. + */ + public static synchronized void stop() { + if (instance != null) { + instance.compose.stop(); + instance = null; + } + } + + /** + * Returns the running runtime. + * + * @return the suite-scoped runtime + * @throws IllegalStateException if the stack has not been started + */ + public static synchronized TckRuntime get() { + if (instance == null) { + throw new IllegalStateException("TCK runtime has not been started"); + } + return instance; + } + + /** + * Returns the provider author's harness. + * + * @return the discovered harness + */ + public ProviderTckHarness harness() { + return harness; + } + + /** + * Returns the client for the backend's control API. + * + * @return the control API client + */ + public ControlApiClient controlApi() { + return controlApi; + } + + /** + * Returns the host and mapped ports of the running stack. + * + * @return the backend endpoint + */ + public BackendEndpoint endpoint() { + return endpoint; + } + + private static ComposeContainer startCompose(ProviderTckHarness harness) { + File composeFile = harness.composeFile(); + if (!composeFile.isFile()) { + throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() + + ". ProviderTckHarness.composeFile() is resolved relative to the module directory."); + } + ComposeContainer compose = new ComposeContainer(composeFile); + + compose.withExposedService(harness.backendService(), harness.controlPort(), Wait.forListeningPort()); + for (Integer port : harness.backendPorts()) { + compose.withExposedService(harness.backendService(), port, Wait.forListeningPort()); + } + for (Map.Entry> service : + harness.additionalExposedPorts().entrySet()) { + for (Integer port : service.getValue()) { + compose.withExposedService(service.getKey(), port, Wait.forListeningPort()); + } + } + compose.withStartupTimeout(harness.startupTimeout()); + + log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); + compose.start(); + return compose; + } + + /** + * Finds the harness for the suite that is currently executing. + * + *

Primary mechanism: the executing suite class itself, reported by {@link TckSuiteListener}. + * A suite class implements {@link ProviderTckHarness}, so a provider with several transports + * writes one suite class per transport and needs no registration, no system property and no + * build configuration to keep them apart. + * + *

Fallback: {@link java.util.ServiceLoader}, for setups where the launcher does not + * auto-register listeners. That path cannot distinguish between several registered harnesses, + * so it accepts exactly one unless {@link #HARNESS_SELECTOR_PROPERTY} names which to use. + */ + private static ProviderTckHarness discoverHarness() { + Optional> suite = TckSuiteListener.currentSuite(); + if (suite.isPresent()) { + return instantiate(suite.get()); + } + + List found = new ArrayList<>(); + ServiceLoader.load(ProviderTckHarness.class).forEach(found::add); + + if (found.isEmpty()) { + throw new IllegalStateException("No ProviderTckHarness found. Write a test class extending " + + "AbstractProviderTckTest; it is both the JUnit suite and the harness."); + } + if (found.size() == 1) { + return found.get(0); + } + + String selector = System.getProperty(HARNESS_SELECTOR_PROPERTY); + if (selector == null) { + throw new IllegalStateException("Several ProviderTckHarness implementations are registered (" + + found.stream().map(h -> h.getClass().getSimpleName()).collect(Collectors.joining(", ")) + + ") and the executing suite could not be determined, which normally means the JUnit " + + "Platform did not auto-register TckSuiteListener. Select one with -D" + + HARNESS_SELECTOR_PROPERTY + "=."); + } + return found.stream() + .filter(h -> h.getClass().getSimpleName().equals(selector)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No registered ProviderTckHarness named '" + selector + + "'. Registered: " + + found.stream().map(h -> h.getClass().getSimpleName()).collect(Collectors.joining(", ")))); + } + + private static ProviderTckHarness instantiate(Class suite) { + try { + return suite.getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + suite.getName() + " could not be instantiated. A TCK suite class needs a public no-argument " + + "constructor, because the TCK creates one to read its configuration.", + e); + } + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckState.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckState.java new file mode 100644 index 000000000..7f911cc1f --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckState.java @@ -0,0 +1,64 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.Client; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.MutableContext; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Scenario-scoped mutable state, injected into every step definition class by PicoContainer. + * + *

One instance per scenario. Anything that must survive across scenarios — the Compose stack, + * the control API client, the discovered harness — lives in {@link TckRuntime} instead. + */ +@SuppressFBWarnings( + value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD", + justification = "Intentional mutable state sharing required by Cucumber PicoContainer DI") +public class TckState { + + /** Client bound to the domain the provider under test is registered under. */ + public Client client; + + /** The provider under test. */ + public FeatureProvider provider; + + /** Scenario-scoped OpenFeature domain, so scenarios cannot see each other's providers. */ + public String domain; + + /** The flag the current scenario is exercising. */ + public FlagUnderTest flag; + + /** Evaluation context accumulated by the context steps. */ + public MutableContext context = new MutableContext(); + + /** Result of the most recent evaluation. */ + public FlagEvaluationDetails evaluation; + + /** + * A previously resolved value, captured so a later evaluation can be asserted to differ. + * + *

Used by the configuration-change scenario. Asserting "the value changed" rather than "the + * value is now X" keeps the scenario portable: the control API only requires that + * {@code POST /change} changes the resolved value of {@code changing-flag}, not which concrete + * value it changes to. + */ + public Object rememberedValue; + + /** + * Any exception thrown out of the most recent evaluation call. + * + *

The SDK contract is that typed evaluation never throws — errors surface as an error code + * and the code default. The evaluation step records rather than propagates, so a scenario can + * assert this explicitly instead of a thrown exception merely showing up as a step failure. + */ + public RuntimeException evaluationException; + + /** Events observed by handlers registered in this scenario. */ + public final ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>(); + + /** The event most recently matched by an await step. */ + public Optional lastEvent = Optional.empty(); +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckSuiteListener.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckSuiteListener.java new file mode 100644 index 000000000..31251ed3f --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckSuiteListener.java @@ -0,0 +1,86 @@ +package dev.openfeature.contrib.tools.providertck; + +import java.lang.reflect.Modifier; +import java.util.Optional; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks which TCK suite is currently executing, so the step definitions can find its harness. + * + *

The problem this solves: Cucumber's {@code @BeforeAll} is static and carries no information + * about which suite triggered it. A provider with more than one transport — flagd has RPC and + * in-process — therefore has no way to tell the glue which of its harnesses to use. Discovering + * harnesses through {@link java.util.ServiceLoader} alone makes that ambiguous the moment a second + * one is registered, and resolving it with a system property would push a separate Surefire + * execution per mode onto every adopter's POM. + * + *

Instead: a concrete suite class is a {@link ProviderTckHarness}, and the JUnit + * Platform tells us which one is running. This listener watches for a container whose source is a + * concrete class implementing the SPI and records it for the duration of that suite's execution. + * Adding a second mode is then a second class and nothing else — no registration file, no system + * property, no build configuration. + * + *

Registered through {@code META-INF/services/org.junit.platform.launcher.TestExecutionListener} + * inside this JAR, so it is picked up automatically by Surefire, Gradle and IDEs. It ignores every + * container that is not a TCK suite, so it is inert in builds that do not use the TCK. + */ +public class TckSuiteListener implements TestExecutionListener { + + private static final Logger log = LoggerFactory.getLogger(TckSuiteListener.class); + + private static volatile Class current; + + @Override + public void executionStarted(TestIdentifier testIdentifier) { + harnessClassOf(testIdentifier).ifPresent(suite -> { + current = suite; + log.debug("TCK suite started: {}", suite.getName()); + }); + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult result) { + harnessClassOf(testIdentifier).ifPresent(suite -> { + if (suite.equals(current)) { + current = null; + } + }); + } + + /** + * Returns the suite class currently executing, if it is a TCK suite. + * + * @return the executing suite class, or empty when none is running or the listener was not + * registered + */ + static Optional> currentSuite() { + return Optional.ofNullable(current); + } + + private static Optional> harnessClassOf(TestIdentifier testIdentifier) { + return testIdentifier + .getSource() + .filter(ClassSource.class::isInstance) + .map(ClassSource.class::cast) + .flatMap(TckSuiteListener::loadClass) + .filter(ProviderTckHarness.class::isAssignableFrom) + .filter(candidate -> !Modifier.isAbstract(candidate.getModifiers())) + .map(candidate -> candidate.asSubclass(ProviderTckHarness.class)); + } + + private static Optional> loadClass(ClassSource source) { + try { + // ClassSource resolves the class lazily and throws when it cannot be loaded — which is + // routine for sources belonging to other engines, so it must not fail the run. + return Optional.of(source.getJavaClass()); + } catch (RuntimeException e) { + log.trace("Ignoring unloadable class source {}", source, e); + return Optional.empty(); + } + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckValues.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckValues.java new file mode 100644 index 000000000..7386e977c --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckValues.java @@ -0,0 +1,58 @@ +package dev.openfeature.contrib.tools.providertck; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.Value; +import java.io.IOException; + +/** + * Converts the string values written in feature files into the typed Java values the SDK expects. + * + *

Gherkin has no type system — every cell in an Examples table is a string. The declared flag + * type in the step is therefore the only thing that distinguishes an integer flag from a float + * flag, and this class is where that distinction is made real. {@code Integer} produces an + * {@link Integer}; {@code Float} produces a {@link Double}. Nothing widens one into the other. + */ +public final class TckValues { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private TckValues() {} + + /** + * Converts a feature-file string to a typed value. + * + * @param value the raw string from the feature file; the literal {@code null} yields + * {@code null} + * @param type the declared type, one of {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @return the converted value + */ + public static Object convert(String value, String type) { + if ("null".equals(value)) { + return null; + } + switch (type) { + case "Boolean": + return Boolean.parseBoolean(value); + case "String": + return value; + case "Integer": + return Integer.parseInt(value); + case "Float": + return Double.parseDouble(value); + case "Object": + return toValue(value); + default: + throw new IllegalArgumentException("Unknown flag type '" + type + + "'. Supported types are Boolean, String, Integer, Float and Object."); + } + } + + private static Value toValue(String json) { + try { + return Value.objectToValue(MAPPER.readValue(json, Object.class)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not parse '" + json + "' as an Object flag value", e); + } + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java new file mode 100644 index 000000000..526068cb4 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java @@ -0,0 +1,39 @@ +package dev.openfeature.contrib.tools.providertck.steps; + +import dev.openfeature.contrib.tools.providertck.ProviderTckHarness; +import dev.openfeature.contrib.tools.providertck.TckRuntime; +import dev.openfeature.contrib.tools.providertck.TckState; + +/** + * Base for the TCK step definition classes. + * + *

Holds the PicoContainer-injected scenario state and gives subclasses convenience access to the + * suite-scoped runtime. + */ +public abstract class AbstractSteps { + + /** Scenario-scoped state, shared across all step classes in a scenario. */ + protected final TckState state; + + protected AbstractSteps(TckState state) { + this.state = state; + } + + /** + * Returns the suite-scoped runtime that owns the Compose stack and control API. + * + * @return the running TCK runtime + */ + protected TckRuntime runtime() { + return TckRuntime.get(); + } + + /** + * Returns the provider author's harness. + * + * @return the discovered harness + */ + protected ProviderTckHarness harness() { + return TckRuntime.get().harness(); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ContextSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ContextSteps.java new file mode 100644 index 000000000..b706086b2 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ContextSteps.java @@ -0,0 +1,71 @@ +package dev.openfeature.contrib.tools.providertck.steps; + +import dev.openfeature.contrib.tools.providertck.TckState; +import dev.openfeature.sdk.MutableStructure; +import io.cucumber.java.en.Given; + +/** + * Steps that build the evaluation context passed to the evaluation call. + * + *

Step vocabulary is inherited verbatim from the flagd test harness. + * + *

Note the TCK cannot currently assert that the context reached the backend intact. + * Doing so needs an echo operation on the control API — something like + * {@code GET /last-evaluation} returning the request the backend last received — which the control + * API does not yet define. Context passthrough is therefore a known gap rather than a covered case. + */ +public class ContextSteps extends AbstractSteps { + + public ContextSteps(TckState state) { + super(state); + } + + /** + * Adds a typed entry to the evaluation context. + * + * @param key the context key + * @param type one of {@code Boolean}, {@code String}, {@code Integer} or {@code Float} + * @param value the value, as written in the feature file + */ + @Given("a context containing a key {string}, with type {string} and with value {string}") + public void contextContainingKeyWithTypeAndValue(String key, String type, String value) { + switch (type) { + case "Boolean": + state.context.add(key, Boolean.parseBoolean(value)); + break; + case "Integer": + state.context.add(key, Integer.parseInt(value)); + break; + case "Float": + state.context.add(key, Double.parseDouble(value)); + break; + case "String": + state.context.add(key, value); + break; + default: + throw new IllegalArgumentException("Unknown context value type '" + type + "'"); + } + } + + /** + * Sets the targeting key on the evaluation context. + * + * @param targetingKey the targeting key + */ + @Given("a context containing a targeting key with value {string}") + public void contextContainingTargetingKey(String targetingKey) { + state.context.setTargetingKey(targetingKey); + } + + /** + * Adds a nested structure entry to the evaluation context. + * + * @param outerKey the outer key + * @param innerKey the key inside the nested structure + * @param value the string value stored under the inner key + */ + @Given("a context containing a nested property with outer key {string} and inner key {string}, with value {string}") + public void contextContainingNestedProperty(String outerKey, String innerKey, String value) { + state.context.add(outerKey, new MutableStructure().add(innerKey, value)); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/EventSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/EventSteps.java new file mode 100644 index 000000000..6bc28b6c8 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/EventSteps.java @@ -0,0 +1,119 @@ +package dev.openfeature.contrib.tools.providertck.steps; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.awaitility.Awaitility.await; + +import dev.openfeature.contrib.tools.providertck.ProviderEventRecord; +import dev.openfeature.contrib.tools.providertck.ProviderTckHarness; +import dev.openfeature.contrib.tools.providertck.TckState; +import dev.openfeature.sdk.ProviderEvent; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registration of provider event handlers and awaiting the events they observe. + * + *

Step vocabulary is inherited verbatim from the flagd test harness. The one behavioural change + * is that the default await timeout comes from {@link ProviderTckHarness#eventTimeout()} instead of + * being a hard-coded constant, because how fast a provider notices a backend change differs by + * orders of magnitude between streaming and polling transports. + */ +public class EventSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(EventSteps.class); + + public EventSteps(TckState state) { + super(state); + } + + /** + * Registers a handler for one kind of provider event. + * + * @param eventType one of {@code ready}, {@code error}, {@code stale} or {@code change} + */ + @Given("a {} event handler") + public void registerEventHandler(String eventType) { + state.client.on(mapEventType(eventType), details -> { + log.info("{} event observed", eventType); + state.events.add(new ProviderEventRecord(eventType, details)); + }); + } + + /** + * Awaits an event of the given kind, using the provider's configured timeout. + * + * @param eventType the event kind + */ + @When("a {} event was fired") + public void eventWasFired(String eventType) { + awaitEvent(eventType, harness().eventTimeout().toMillis()); + } + + /** + * Awaits an event of the given kind, using the provider's configured timeout. + * + * @param eventType the event kind + */ + @Then("the {} event handler should have been executed") + public void theEventHandlerShouldHaveBeenExecuted(String eventType) { + awaitEvent(eventType, harness().eventTimeout().toMillis()); + } + + /** + * Awaits an event of the given kind within an explicit deadline. + * + *

Use this where the deadline is part of what the scenario asserts — for instance that a + * provider initialised against a dead backend reports the failure promptly rather than hanging. + * The explicit value always wins over {@link ProviderTckHarness#eventTimeout()}. + * + * @param eventType the event kind + * @param milliseconds the deadline + */ + @Then("the {} event handler should have been executed within {int}ms") + public void theEventHandlerShouldHaveBeenExecutedWithin(String eventType, int milliseconds) { + awaitEvent(eventType, milliseconds); + } + + private void awaitEvent(String eventType, long milliseconds) { + log.info("Awaiting {} event (timeout {}ms)", eventType, milliseconds); + await().alias("provider event " + eventType) + .atMost(milliseconds, MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> + state.events.stream().anyMatch(event -> event.type().equals(eventType))); + + // Drain up to and including the first match. Without this, a READY recorded before a + // disconnect would satisfy a later assertion expecting a *new* READY after reconnect, + // and the reconnect scenarios would pass without the provider ever reconnecting. + // Events that arrived after the match are preserved for subsequent steps. + ProviderEventRecord matched = null; + while (!state.events.isEmpty()) { + ProviderEventRecord head = state.events.poll(); + if (head != null && head.type().equals(eventType)) { + matched = head; + break; + } + } + state.lastEvent = Optional.ofNullable(matched); + } + + private static ProviderEvent mapEventType(String eventType) { + switch (eventType) { + case "ready": + return ProviderEvent.PROVIDER_READY; + case "error": + return ProviderEvent.PROVIDER_ERROR; + case "stale": + return ProviderEvent.PROVIDER_STALE; + case "change": + return ProviderEvent.PROVIDER_CONFIGURATION_CHANGED; + default: + throw new IllegalArgumentException( + "Unknown event type '" + eventType + "'. The TCK recognises ready, error, stale and change."); + } + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/FlagSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/FlagSteps.java new file mode 100644 index 000000000..e8df0b0fe --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/FlagSteps.java @@ -0,0 +1,255 @@ +package dev.openfeature.contrib.tools.providertck.steps; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.openfeature.contrib.tools.providertck.FlagUnderTest; +import dev.openfeature.contrib.tools.providertck.ProviderEventRecord; +import dev.openfeature.contrib.tools.providertck.TckState; +import dev.openfeature.contrib.tools.providertck.TckValues; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.Structure; +import dev.openfeature.sdk.Value; +import io.cucumber.datatable.DataTable; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Flag evaluation steps and assertions on the resulting resolution details. + * + *

Step vocabulary is inherited verbatim from the flagd test harness, which was already + * provider-neutral here. + */ +public class FlagSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(FlagSteps.class); + + public FlagSteps(TckState state) { + super(state); + } + + /** + * Declares the flag the scenario will evaluate. + * + * @param type the declared type: {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @param key the flag key + * @param defaultValue the code default, as written in the feature file + */ + @Given("a {}-flag with key {string} and a default value {string}") + public void flagWithKeyAndDefaultValue(String type, String key, String defaultValue) { + state.flag = new FlagUnderTest(key, type, TckValues.convert(defaultValue, type)); + } + + /** + * Evaluates the declared flag through the typed API matching its declared type. + * + *

Dispatch is on the declared type alone, which is what makes the integer/float distinction + * observable: an {@code Integer} flag goes through {@code getIntegerDetails} and a {@code Float} + * flag through {@code getDoubleDetails}, with no widening in between. A provider that returns a + * double for an integer flag fails here rather than being quietly accommodated. + * + *

Exceptions are recorded rather than propagated. The SDK contract is that typed evaluation + * never throws — errors surface as an error code plus the code default — so + * {@code no exception should have been thrown} can assert that explicitly instead of the + * scenario merely erroring out. + */ + @When("the flag was evaluated with details") + public void theFlagWasEvaluatedWithDetails() { + FlagUnderTest flag = state.flag; + try { + switch (flag.type()) { + case "Boolean": + state.evaluation = + state.client.getBooleanDetails(flag.key(), (Boolean) flag.defaultValue(), state.context); + break; + case "String": + state.evaluation = + state.client.getStringDetails(flag.key(), (String) flag.defaultValue(), state.context); + break; + case "Integer": + state.evaluation = + state.client.getIntegerDetails(flag.key(), (Integer) flag.defaultValue(), state.context); + break; + case "Float": + state.evaluation = + state.client.getDoubleDetails(flag.key(), (Double) flag.defaultValue(), state.context); + break; + case "Object": + state.evaluation = + state.client.getObjectDetails(flag.key(), (Value) flag.defaultValue(), state.context); + break; + default: + throw new IllegalArgumentException("Unknown flag type '" + flag.type() + "'"); + } + } catch (RuntimeException e) { + log.warn("Evaluation of '{}' threw, which violates the SDK contract", flag.key(), e); + state.evaluationException = e; + } + } + + /** + * Asserts the resolved value, converted according to the flag's declared type. + * + * @param value the expected value, as written in the feature file + */ + @Then("the resolved details value should be \"{}\"") + public void theResolvedDetailsValueShouldBe(String value) { + requireEvaluation(); + if (state.evaluation.getErrorCode() != null) { + log.info( + "Evaluation of '{}' carries error code {}: {}", + state.flag.key(), + state.evaluation.getErrorCode(), + state.evaluation.getErrorMessage()); + } + assertThat(state.evaluation.getValue()).isEqualTo(TckValues.convert(value, state.flag.type())); + } + + /** + * Asserts the resolution reason. + * + * @param reason the expected reason + */ + @Then("the reason should be {string}") + public void theReasonShouldBe(String reason) { + requireEvaluation(); + assertThat(state.evaluation.getReason()).isEqualTo(reason); + } + + /** + * Asserts the resolved variant. + * + * @param variant the expected variant + */ + @Then("the variant should be {string}") + public void theVariantShouldBe(String variant) { + requireEvaluation(); + assertThat(state.evaluation.getVariant()).isEqualTo(variant); + } + + /** + * Asserts the error code, where an empty string means no error. + * + * @param errorCode the expected {@link ErrorCode} name, or an empty string + */ + @Then("the error-code should be {string}") + public void theErrorCodeShouldBe(String errorCode) { + requireEvaluation(); + if (errorCode == null || errorCode.isEmpty()) { + assertThat(state.evaluation.getErrorCode()).isNull(); + } else { + assertThat(state.evaluation.getErrorCode()).isEqualTo(ErrorCode.valueOf(errorCode)); + } + } + + /** + * Captures the current resolved value so a later evaluation can be asserted to differ. + * + *

Added by the TCK, for the configuration-change scenario. The control API only requires + * that {@code POST /change} changes the resolved value of {@code changing-flag}; which concrete + * value it changes to is vendor-defined. Asserting a delta rather than an absolute keeps the + * scenario portable and independent of how many times it has run against the same stack. + */ + @When("the resolved value is remembered") + public void theResolvedValueIsRemembered() { + requireEvaluation(); + state.rememberedValue = state.evaluation.getValue(); + } + + /** + * Asserts that re-evaluation produced a different value than the remembered one. + */ + @Then("the resolved details value should have changed") + public void theResolvedDetailsValueShouldHaveChanged() { + requireEvaluation(); + assertThat(state.evaluation.getValue()) + .withFailMessage( + "Expected the value of '%s' to differ after the configuration change, " + + "but it is still %s. The provider signalled the change but did not apply it.", + state.flag.key(), state.rememberedValue) + .isNotEqualTo(state.rememberedValue); + } + + /** + * Asserts that a resolved structure contains the given entries. + * + *

Table columns are {@code key}, {@code type} and {@code value}, mirroring the shape of the + * flagd harness's metadata table. Asserting individual entries rather than a whole JSON blob + * keeps the step readable and avoids quoting a JSON document inside a Gherkin cell. + * + * @param expected a table of expected entries + */ + @Then("the resolved object value should contain") + public void theResolvedObjectValueShouldContain(DataTable expected) { + requireEvaluation(); + assertThat(state.evaluation.getValue()) + .as("resolved value of '%s' is a structure", state.flag.key()) + .isInstanceOf(Value.class); + Structure structure = ((Value) state.evaluation.getValue()).asStructure(); + assertThat(structure) + .as("resolved value of '%s' is a structure", state.flag.key()) + .isNotNull(); + + for (Map row : expected.asMaps()) { + String key = row.get("key"); + Value actual = structure.getValue(key); + assertThat(actual).as("structure entry '%s'", key).isNotNull(); + + Object expectedValue = TckValues.convert(row.get("value"), row.get("type")); + Object actualValue = actual.asObject(); + + // Numbers nested inside a structure are compared by value rather than by Java type. + // Structures arrive as JSON, and JSON has a single number type — whether 100 comes + // back as an Integer or a Double is an artefact of the provider's JSON library, not + // an observable part of the provider contract. The integer/float distinction that + // *is* part of the contract applies to top-level typed evaluation, and is asserted + // by the dedicated scenarios in evaluation.feature and errors.feature. + if (expectedValue instanceof Number && actualValue instanceof Number) { + assertThat(((Number) actualValue).doubleValue()) + .as("structure entry '%s'", key) + .isEqualTo(((Number) expectedValue).doubleValue()); + } else { + assertThat(actualValue).as("structure entry '%s'", key).isEqualTo(expectedValue); + } + } + } + + /** + * Asserts that the evaluation returned normally. + * + *

Added by the TCK. The spec requires typed evaluation to absorb every error into the + * returned details, so an error scenario must prove both halves: the right error code, and no + * exception escaping to the caller. + */ + @Then("no exception should have been thrown") + public void noExceptionShouldHaveBeenThrown() { + assertThat(state.evaluationException) + .withFailMessage( + "Evaluation threw %s, but typed evaluation must never throw — " + + "errors belong in the resolution details.", + state.evaluationException) + .isNull(); + } + + /** + * Asserts the flag under test appears in the payload of the most recently matched event. + */ + @Then("the flag should be part of the event payload") + public void theFlagShouldBePartOfTheEventPayload() { + ProviderEventRecord event = state.lastEvent.orElseThrow( + () -> new AssertionError("No event has been matched yet; await an event before asserting its payload")); + assertThat(event.details().getFlagsChanged()).contains(state.flag.key()); + } + + private void requireEvaluation() { + if (state.evaluation == null) { + throw new AssertionError("No evaluation has been performed. " + + "Did the scenario forget 'When the flag was evaluated with details'?" + + (state.evaluationException == null ? "" : " Evaluation threw: " + state.evaluationException)); + } + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java new file mode 100644 index 000000000..8fe239414 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java @@ -0,0 +1,216 @@ +package dev.openfeature.contrib.tools.providertck.steps; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.awaitility.Awaitility.await; + +import dev.openfeature.contrib.tools.providertck.Capability; +import dev.openfeature.contrib.tools.providertck.ProviderTckHarness; +import dev.openfeature.contrib.tools.providertck.TckRuntime; +import dev.openfeature.contrib.tools.providertck.TckState; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.NoOpProvider; +import dev.openfeature.sdk.OpenFeatureAPI; +import dev.openfeature.sdk.ProviderState; +import io.cucumber.java.After; +import io.cucumber.java.AfterAll; +import io.cucumber.java.Before; +import io.cucumber.java.BeforeAll; +import io.cucumber.java.Scenario; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import org.opentest4j.TestAbortedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Lifecycle and control API steps: bringing the Compose stack up, gating scenarios on declared + * capabilities, creating and registering the provider under test, and simulating backend outages. + * + *

Step vocabulary is inherited from the flagd test harness so that existing feature files port + * with a near-zero diff. The only change is dropping the word {@code flagd} from the provider setup + * step: {@code a stable flagd provider} becomes {@code a stable provider}. + */ +public class ProviderSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(ProviderSteps.class); + + public ProviderSteps(TckState state) { + super(state); + } + + /** + * Starts the Compose stack once, before the first scenario. + */ + @BeforeAll + public static void beforeAll() { + TckRuntime.startIfNeeded(); + } + + /** + * Stops the Compose stack after the last scenario. + */ + @AfterAll + public static void afterAll() { + TckRuntime.stop(); + } + + /** + * Skips scenarios that exercise a capability the provider did not declare. + * + *

Aborting rather than failing means the scenario is reported as skipped by the + * JUnit Platform. That distinction is the whole point: a provider that does not support + * configuration-change events should see those scenarios visibly excluded, never silently + * green. + * + * @param scenario the scenario about to run + */ + @Before(order = 0) + public void gateOnCapabilities(Scenario scenario) { + Set supported = harness().capabilities(); + for (String tag : scenario.getSourceTagNames()) { + Optional capability = Capability.fromTag(tag); + if (capability.isPresent() && !supported.contains(capability.get())) { + throw new TestAbortedException("Skipped: provider does not declare capability " + + capability.get().name() + " (tag " + tag + "). Declared capabilities: " + supported); + } + } + } + + /** + * Restores the backend to a running, freshly seeded state before each scenario. + * + *

Scenario isolation is achieved here, through the control API, and never by restarting + * containers — see the no-container-restart invariant in {@code openapi/control-api.yaml}. + */ + @Before(order = 10) + public void prepareBackend() { + ProviderTckHarness harness = harness(); + runtime().controlApi().prepareScenario(harness.defaultConfig()); + } + + /** + * Tears the provider down without disturbing the Compose stack. + * + *

Replaces the domain's provider with a {@link NoOpProvider} through the SDK lifecycle rather + * than calling {@code shutdown()} directly, because only the former makes the SDK detach the + * event provider and shut down its emitter executor. Skipping this leaks an emitter thread per + * scenario and lets events from a finished scenario surface in the next one. + */ + @After + public void tearDown() { + if (state.domain != null) { + OpenFeatureAPI.getInstance().setProvider(state.domain, new NoOpProvider()); + } + } + + /** + * Creates the provider under test and registers it under a scenario-scoped domain. + * + *

Two provider flavours are recognised. A {@code stable} provider is built by the harness + * against the running stack and registered with {@code setProviderAndWait}, so the step does not + * return until the provider is ready. An {@code unavailable} provider points at a dead backend + * and is registered with {@code setProvider}, deliberately without waiting — the scenario's + * whole point is that readiness never arrives. + * + * @param flavour either {@code stable} or {@code unavailable} + */ + @Given("a {} provider") + public void createProvider(String flavour) { + ProviderTckHarness harness = harness(); + FeatureProvider provider; + boolean waitForReady; + + switch (flavour) { + case "stable": + provider = harness.createProvider(runtime().endpoint()); + waitForReady = true; + break; + case "unavailable": + provider = harness.createUnavailableProvider(); + waitForReady = false; + break; + default: + throw new IllegalArgumentException( + "Unknown provider flavour '" + flavour + "'. The TCK recognises 'stable' and 'unavailable'."); + } + + String domain = "tck-" + UUID.randomUUID(); + OpenFeatureAPI api = OpenFeatureAPI.getInstance(); + if (waitForReady) { + api.setProviderAndWait(domain, provider); + } else { + api.setProvider(domain, provider); + } + + state.provider = provider; + state.domain = domain; + state.client = api.getClient(domain); + log.info( + "Registered {} provider {} under domain {}", + flavour, + provider.getMetadata().getName(), + domain); + } + + /** + * Makes the backend unreachable for the rest of the scenario. + */ + @When("the connection is lost") + public void theConnectionIsLost() { + runtime().controlApi().stop(); + } + + /** + * Makes the backend unreachable for a bounded period, then brings it back. + * + * @param seconds how long the backend stays unreachable + */ + @When("the connection is lost for {int}s") + public void theConnectionIsLostFor(int seconds) { + runtime().controlApi().restart(seconds); + } + + /** + * Brings the backend back after {@code the connection is lost}. + * + *

Added by the TCK. The flagd harness only has the self-healing + * {@code the connection is lost for {int}s} form, which cannot express "assert the provider is + * stale, and only then reconnect" — the reconnect races the assertion. Splitting the outage + * into an explicit start and end makes the stale-then-ready transition deterministic. + */ + @When("the connection is restored") + public void theConnectionIsRestored() { + runtime().controlApi().start(harness().defaultConfig()); + } + + /** + * Mutates flag configuration so a conforming provider observes a configuration change. + */ + @When("the flag was modified") + public void theFlagWasModified() { + runtime().controlApi().change(); + } + + /** + * Asserts the provider settles into the expected lifecycle state. + * + *

Awaits rather than asserting immediately. State transitions are asynchronous in every + * provider, and how quickly one notices a backend change varies by orders of magnitude between + * streaming and polling transports, so the timeout comes from + * {@link ProviderTckHarness#readyTimeout()}. + * + * @param expected the expected {@link ProviderState}, case-insensitive + */ + @Then("the client should be in {} state") + public void theClientShouldBeInState(String expected) { + ProviderState target = ProviderState.valueOf(expected.toUpperCase()); + await().alias("provider state " + target) + .atMost(harness().readyTimeout().toMillis(), MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> state.client.getProviderState() == target); + } +} diff --git a/tools/provider-tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/tools/provider-tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 000000000..407c53b6a --- /dev/null +++ b/tools/provider-tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +dev.openfeature.contrib.tools.providertck.TckSuiteListener diff --git a/tools/provider-tck/src/main/resources/features/errors.feature b/tools/provider-tck/src/main/resources/features/errors.feature new file mode 100644 index 000000000..0346df3da --- /dev/null +++ b/tools/provider-tck/src/main/resources/features/errors.feature @@ -0,0 +1,80 @@ +Feature: Provider error handling + + # Every scenario here asserts the same three-part contract, because all three parts matter and + # providers routinely get one of them wrong: + # + # 1. the code default is returned — an application must keep working, + # 2. the correct error code is reported — an application must be able to tell what went wrong, + # 3. nothing is thrown — an unhandled exception from a flag evaluation is never acceptable. + # + # Requires the backend to be seeded with the canonical flag set — see flags/canonical-flags.json. + + Background: + Given a stable provider + + Scenario Outline: Requesting the wrong type returns the code default + # The full non-numeric mismatch matrix. Numeric coercion is a separate question and is covered + # by the @strict-numeric-typing scenarios below, because "is 0.5 an integer?" has a defensible + # wrong answer whereas "is a string a boolean?" does not. + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: a string flag requested as something else + | key | requested | default | + | string-flag | Boolean | false | + | string-flag | Integer | 1 | + | string-flag | Float | 0.1 | + | wrong-flag | Boolean | false | + + Examples: a boolean flag requested as something else + | key | requested | default | + | boolean-flag | String | fallback | + | boolean-flag | Integer | 1 | + | boolean-flag | Float | 0.1 | + + Examples: a numeric flag requested as a non-numeric type + | key | requested | default | + | integer-flag | Boolean | false | + | integer-flag | String | fallback | + | float-flag | Boolean | false | + | float-flag | String | fallback | + + @object + Scenario Outline: Requesting a structured flag as a scalar returns the code default + Given a -flag with key "object-flag" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: + | requested | default | + | Boolean | false | + | String | fallback | + | Integer | 1 | + | Float | 0.1 | + + @strict-numeric-typing + Scenario: A float flag is not silently narrowed to an integer + # 'float-flag' resolves to 0.5. Narrowing that to an integer would lose information + # silently, so it must be reported as a type mismatch rather than rounded. + Given a Integer-flag with key "float-flag" and a default value "1" + When the flag was evaluated with details + Then the resolved details value should be "1" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Scenario: An unknown flag key returns the code default + # 'missing-flag' is deliberately absent from the canonical flag set. + Given a String-flag with key "missing-flag" and a default value "fallback" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + And the reason should be "ERROR" + And the error-code should be "FLAG_NOT_FOUND" + And no exception should have been thrown diff --git a/tools/provider-tck/src/main/resources/features/evaluation.feature b/tools/provider-tck/src/main/resources/features/evaluation.feature new file mode 100644 index 000000000..e89f174a5 --- /dev/null +++ b/tools/provider-tck/src/main/resources/features/evaluation.feature @@ -0,0 +1,59 @@ +Feature: Provider flag evaluation + + # Verifies that a provider maps backend responses onto typed resolution details correctly. + # + # This does NOT test the backend's evaluation logic. Every flag in the canonical set resolves + # to its default variant with no targeting involved, so what is under test is purely the + # provider's mapping of a backend response to a value, a variant and a reason. + # + # Requires the backend to be seeded with the canonical flag set — see flags/canonical-flags.json. + + Background: + Given a stable provider + + Scenario Outline: Resolve values with variant and reason + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the variant should be "" + And the reason should be "" + And the error-code should be "" + And no exception should have been thrown + + Examples: + | key | type | default | value | variant | reason | + | boolean-flag | Boolean | false | true | on | STATIC | + | string-flag | String | bye | hi | greeting | STATIC | + | integer-flag | Integer | 1 | 10 | ten | STATIC | + | float-flag | Float | 0.1 | 0.5 | half | STATIC | + + Scenario: An integer flag resolves as an integer + # Paired with the float scenario below and with the narrowing scenario in errors.feature. + # Together they pin down that the two numeric types stay distinct rather than both being + # funnelled through one numeric representation. + Given a Integer-flag with key "integer-flag" and a default value "1" + When the flag was evaluated with details + Then the resolved details value should be "10" + And the error-code should be "" + And no exception should have been thrown + + Scenario: A float flag resolves as a float + Given a Float-flag with key "float-flag" and a default value "0.1" + When the flag was evaluated with details + Then the resolved details value should be "0.5" + And the error-code should be "" + And no exception should have been thrown + + @object + Scenario: Resolve a structured value + Given a Object-flag with key "object-flag" and a default value "{}" + When the flag was evaluated with details + Then the variant should be "template" + And the reason should be "STATIC" + And the error-code should be "" + And no exception should have been thrown + And the resolved object value should contain + | key | type | value | + | showImages | Boolean | true | + | title | String | Check out these pics! | + | imagesPerPage | Integer | 100 | diff --git a/tools/provider-tck/src/main/resources/features/events.feature b/tools/provider-tck/src/main/resources/features/events.feature new file mode 100644 index 000000000..00e7e5ef6 --- /dev/null +++ b/tools/provider-tck/src/main/resources/features/events.feature @@ -0,0 +1,42 @@ +@events +Feature: Provider events + + # Verifies that a provider notices changes in its backend and both signals them and acts on + # them. Signalling alone is not enough: a configuration-change event that is not followed by + # a changed evaluation result is a lie, so each scenario asserts the event AND the behaviour. + # + # Outages here are simulated inside the running stack via the control API. No container is + # ever stopped or restarted — see the invariant in openapi/control-api.yaml. + + Background: + Given a stable provider + + @configuration-change + Scenario: A configuration change is signalled and applied + Given a String-flag with key "changing-flag" and a default value "unset" + And a change event handler + When the flag was evaluated with details + And the resolved value is remembered + And the flag was modified + Then the change event handler should have been executed + And the flag should be part of the event payload + When the flag was evaluated with details + Then the resolved details value should have changed + And no exception should have been thrown + + @stale + Scenario: Losing the backend makes the provider stale, regaining it makes it ready again + Given a ready event handler + And a stale event handler + When a ready event was fired + And the connection is lost + Then the stale event handler should have been executed + And the client should be in stale state + When the connection is restored + Then the ready event handler should have been executed + And the client should be in ready state + + # Deliberately NOT covered here: whether a stale provider keeps serving last-known values + # during the outage. That is caching behaviour, which depends on whether the provider holds a + # local copy of the ruleset, and it belongs behind the @caching capability once those + # scenarios are written. See the "Known gaps" section of the README. diff --git a/tools/provider-tck/src/main/resources/features/lifecycle.feature b/tools/provider-tck/src/main/resources/features/lifecycle.feature new file mode 100644 index 000000000..338b2c052 --- /dev/null +++ b/tools/provider-tck/src/main/resources/features/lifecycle.feature @@ -0,0 +1,40 @@ +@lifecycle +Feature: Provider lifecycle + + # Verifies the two terminal outcomes of provider initialisation: reaching READY against a + # healthy backend, and settling into ERROR against one that cannot be reached. + # + # Gated by @lifecycle rather than @events, and the distinction is load-bearing. Every SDK + # synthesises PROVIDER_READY for a provider that has no initialisation step, so a provider + # without a lifecycle passes the readiness scenario below without demonstrating anything -- + # a NoOpProvider passes it identically. @lifecycle asserts that the provider actually reaches + # its backend during initialisation and that the outcome is observable; a provider that merely + # emits events does not necessarily do that. + # + # The failure case matters more than it looks. A provider that blocks forever, or throws out + # of provider registration, takes the host application down with it — so the requirement is + # not merely that initialisation fails, but that it fails observably and promptly. + + Scenario: A provider reaching its backend becomes ready + Given a stable provider + And a ready event handler + Then the ready event handler should have been executed + And the client should be in ready state + + @unavailable + Scenario: A provider that cannot reach its backend reports an error + Given a unavailable provider + And a error event handler + Then the error event handler should have been executed within 10000ms + And the client should be in error state + + @unavailable + Scenario: A provider that cannot reach its backend still returns code defaults + Given a unavailable provider + And a error event handler + And a Boolean-flag with key "boolean-flag" and a default value "false" + Then the error event handler should have been executed within 10000ms + When the flag was evaluated with details + Then the resolved details value should be "false" + And the reason should be "ERROR" + And no exception should have been thrown diff --git a/tools/provider-tck/src/main/resources/flags/canonical-flags.json b/tools/provider-tck/src/main/resources/flags/canonical-flags.json new file mode 100644 index 000000000..343b3ae52 --- /dev/null +++ b/tools/provider-tck/src/main/resources/flags/canonical-flags.json @@ -0,0 +1,82 @@ +{ + "$comment": [ + "The canonical flag set the TCK's feature files assume. A backend under test MUST serve an", + "equivalent set under the configuration named 'default'.", + "", + "Expressed in the flagd flag-definition format because that is the only widely implemented", + "vendor-neutral format today. The format is not what matters — the keys, types, variant", + "names and resolved values are. Seed them however your backend seeds flags.", + "", + "Two things are load-bearing and easy to get wrong:", + " * 'missing-flag' MUST NOT exist. Its absence is what the FLAG_NOT_FOUND scenario tests.", + " * No flag here has targeting rules. Every scenario expects reason STATIC, because the TCK", + " tests the provider's mapping of a response, not the backend's evaluation logic." + ], + "flags": { + "boolean-flag": { + "state": "ENABLED", + "variants": { + "on": true, + "off": false + }, + "defaultVariant": "on" + }, + "string-flag": { + "state": "ENABLED", + "variants": { + "greeting": "hi", + "parting": "bye" + }, + "defaultVariant": "greeting" + }, + "integer-flag": { + "state": "ENABLED", + "variants": { + "one": 1, + "ten": 10 + }, + "defaultVariant": "ten" + }, + "float-flag": { + "state": "ENABLED", + "variants": { + "tenth": 0.1, + "half": 0.5 + }, + "defaultVariant": "half" + }, + "object-flag": { + "state": "ENABLED", + "variants": { + "empty": {}, + "template": { + "showImages": true, + "title": "Check out these pics!", + "imagesPerPage": 100 + } + }, + "defaultVariant": "template" + }, + "wrong-flag": { + "$comment": "A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario.", + "state": "ENABLED", + "variants": { + "one": "uno", + "two": "dos" + }, + "defaultVariant": "one" + }, + "changing-flag": { + "$comment": [ + "The flag POST /change mutates. The TCK asserts only that its resolved value differs", + "after the change, so which of the two variants you start from does not matter." + ], + "state": "ENABLED", + "variants": { + "foo": "foo", + "bar": "bar" + }, + "defaultVariant": "foo" + } + } +} diff --git a/tools/provider-tck/src/main/resources/openapi/control-api.yaml b/tools/provider-tck/src/main/resources/openapi/control-api.yaml new file mode 100644 index 000000000..fd9bc7000 --- /dev/null +++ b/tools/provider-tck/src/main/resources/openapi/control-api.yaml @@ -0,0 +1,368 @@ +openapi: 3.0.3 + +info: + title: OpenFeature Provider TCK — Backend Control API + version: 0.0.1 + description: | + The control API that a **backend under test** must expose so the OpenFeature + Provider TCK can drive it. + + The TCK verifies the *provider contract*: how a provider maps backend + responses to typed resolution details, lifecycle states and events. To do + that it must be able to put the backend into specific states on demand — + running, unreachable, reconfigured. This document standardises how. + + This specification is derived from the control endpoints already implemented + by [`flagd-testbed`](https://github.com/open-feature/flagd-testbed)'s + "launchpad" server, which is the reference implementation. + + ## Where this document should live + + This file currently ships inside the Java `provider-tck` artifact, but it is + not a Java artifact: it is a language-agnostic contract that every language's + TCK must implement identically, and that backend vendors implement in + whatever language their testbed is written in (Go, for flagd). + + It therefore belongs in the OpenFeature **spec** repository + (`open-feature/spec`), alongside the canonical Gherkin feature files and the + canonical flag set. Those three artifacts are a single unit — a feature file + that evaluates `boolean-flag` is meaningless without the flag definition, and + a disconnect scenario is meaningless without the endpoint that produces the + disconnect. Splitting them across repositories would let them drift. + + Each language's TCK then vendors the spec repo (git submodule or equivalent) + and packages these files into its own distribution format, so that adopting a + TCK never requires a consumer to check out a submodule of their own. + + ## Conformance language + + The key words MUST, MUST NOT, REQUIRED, SHOULD, SHOULD NOT and MAY are to be + interpreted as described in RFC 2119. + + Each operation below is tagged **REQUIRED** or **OPTIONAL**. A backend that + implements every REQUIRED operation can run the full TCK. OPTIONAL operations + have a defined fallback that the TCK applies automatically, so omitting them + costs nothing but precision. + + --- + + ## Normative requirement 1 — the no-container-restart invariant + + > **Container lifecycle operations MUST NOT be used to simulate backend + > unavailability. Backend unavailability MUST be simulated from inside the + > running stack.** + + The TCK starts the vendor's Docker Compose stack **once per test suite** and + reads the dynamically mapped host ports. Testcontainers cannot reliably + preserve mapped ports across a container stop/start in all language + bindings — a restarted container generally comes back on a *different* host + port, which silently invalidates every provider instance already pointed at + the old one. Any TCK implementation in any language hits this, so the + constraint is part of the contract rather than a Java detail. + + Therefore an implementation of `/stop`, `/restart` or any other outage + simulation MUST achieve the outage by one of: + + * killing or suspending the backend **process** inside its container + (the reference behaviour — this is what flagd-testbed does); + * a proxy in the stack refusing or blackholing connections + (e.g. a toxiproxy toxic, an envoy `direct_response`); + * an in-container firewall or socket-level block. + + An implementation MUST NOT `docker stop`, `docker kill`, `docker rm` or + recreate any container in the stack while the suite is running. The stack is + brought up before the first scenario and torn down after the last one, and + the mapped ports MUST remain stable for that entire window. + + --- + + ## Normative requirement 2 — flag state semantics across outages + + Outage simulation and flag-state seeding are orthogonal, and the TCK relies + on that separation for scenario isolation: + + * `POST /start` **MUST** (re)seed flag state to the baseline defined by the + named configuration. Any mutation previously applied by `POST /change` + MUST be discarded. This is what makes `/start` usable as a reset. + * `POST /restart` and a `POST /stop` followed by a `POST /start` **of the + same configuration** MUST leave the backend serving the same baseline + flag state it served before the outage. An outage MUST NOT be observable + as a change in flag *values* — only as a change in *availability*. + * `POST /change` mutations persist until the next `/start` or `/reset`. + + --- + + ## Normative requirement 3 — compose stack conventions + + The backend under test is delivered as a **Docker Compose stack**, not a + single image, so vendors can compose proxies, edge services or several + containers. The TCK only relies on these conventions: + + * One service — by default named `backend`, overridable by the provider + author — exposes the control API on container-internal port `8080` + (also overridable). + * The same stack exposes whatever port(s) the provider connects to. + * **All external ports are dynamically mapped.** A stack MUST NOT pin host + ports; the TCK discovers them after startup and hands them to the + provider factory. + * The stack MAY contain any number of additional services. + + --- + + ## Known gap — evaluation context passthrough + + There is currently no operation for asserting that an evaluation context sent + by the provider actually reached the backend intact. Verifying that requires + an echo mechanism (e.g. `GET /last-evaluation` returning the most recent + request the backend received). Until such an operation exists, context + passthrough is out of scope for the TCK. + + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0 + +servers: + - url: http://{host}:{port} + description: | + Resolved at runtime from the Compose stack. `host` is the Docker host and + `port` is the dynamically mapped host port for the control service's + internal port 8080. + variables: + host: + default: localhost + port: + default: "8080" + +tags: + - name: lifecycle + description: Start and stop the backend process. + - name: availability + description: Simulate outages without touching containers. + - name: flags + description: Seed and mutate flag configuration. + - name: health + description: Readiness of the control API itself. + +paths: + + /start: + post: + tags: [lifecycle] + operationId: start + summary: "[REQUIRED] Start the backend and seed flags to a named baseline" + description: | + Starts the backend process using the named configuration and seeds flag + state to that configuration's baseline. + + MUST be idempotent in the sense that calling it while the backend is + already running is not an error: the implementation restarts the process + (or otherwise ensures it is running) with the requested configuration. + + Because this operation resets flag state, the TCK uses it as its default + scenario-isolation mechanism when `/reset` is not implemented. + + The set of valid configuration names is vendor-defined. Every + implementation MUST support the name `default`, which MUST serve the + canonical flag set the TCK's feature files assume. + + Reference implementation: flagd-testbed launches the `flagd` binary with + the config file of that name from `launchpad/configs` and rewrites + `/flags/allFlags.json`. + parameters: + - name: config + in: query + required: false + description: | + Name of the configuration to start with. Defaults to `default`. + schema: + type: string + default: default + example: default + responses: + "200": + description: Backend started and flag state seeded. + "400": + description: Unknown configuration name. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + /stop: + post: + tags: [availability] + operationId: stop + summary: "[REQUIRED] Make the backend unreachable" + description: | + Makes the backend unreachable to the provider, simulating an outage. + + **MUST NOT stop the container.** See normative requirement 1. The + reference implementation kills the flagd process while its container + keeps running. + + The backend stays unreachable until a subsequent `POST /start`. Calling + `/stop` when the backend is already stopped MUST succeed. + + The TCK uses this to drive providers into `STALE` and `ERROR` states and + to assert `PROVIDER_STALE` / `PROVIDER_ERROR` events. + responses: + "200": + description: Backend is now unreachable; container still running. + + /restart: + post: + tags: [availability] + operationId: restart + summary: "[REQUIRED] Simulate an outage of a bounded duration" + description: | + Makes the backend unreachable, waits `seconds`, then starts it again with + the configuration currently in effect. + + Flag state MUST be preserved across the outage — see normative + requirement 2. This is what distinguishes `/restart` from + `/stop` + `/start`: the former is an availability event, the latter is + also a reset. + + This operation MAY return as soon as the outage has begun rather than + blocking for the full duration; the TCK does not rely on the response + being delayed. It awaits provider events instead. + + The TCK uses this for the disconnect/reconnect scenarios: `STALE` → + `PROVIDER_STALE`, then back to `READY` → `PROVIDER_READY`. + parameters: + - name: seconds + in: query + required: false + description: | + How long the backend stays unreachable. Defaults to 5. + + Providers differ enormously in how fast they notice an outage — + a streaming provider may see it in milliseconds while a polling + provider needs up to a full poll interval. Feature files therefore + parameterise this value and provider authors tune the matching + await timeouts. + schema: + type: integer + format: int32 + minimum: 0 + default: 5 + example: 5 + responses: + "200": + description: Outage started (and, for blocking implementations, ended). + + /change: + post: + tags: [flags] + operationId: change + summary: "[REQUIRED] Mutate flag configuration so the provider observes a change" + description: | + Mutates the flag configuration such that a conforming provider observes a + configuration change and, on re-evaluation, resolves a **different value** + for the affected flag. + + The implementation MUST: + + * change the resolved value of the flag with key `changing-flag`; + * do so without restarting the backend process, so that a provider sees + a configuration-change signal rather than a reconnect; + * make the change durable until the next `/start` or `/reset`. + + The implementation SHOULD toggle between exactly two known values so that + repeated calls are meaningful and the test remains deterministic + regardless of how many times it has run against the same stack. The + reference implementation toggles `changing-flag`'s `defaultVariant` + between `foo` and `bar`. + + The TCK uses this to assert `PROVIDER_CONFIGURATION_CHANGED`, that the + changed flag key appears in the event payload, and that a subsequent + evaluation returns the new value. + responses: + "200": + description: Flag configuration mutated. + + /reset: + post: + tags: [flags] + operationId: reset + summary: "[OPTIONAL] Restore the seeded baseline without an outage" + description: | + Restores flag state to the baseline of the configuration currently in + effect, discarding any mutation applied by `/change`, **without** making + the backend unreachable at any point. + + This is the preferred scenario-isolation primitive: unlike `/start` it + causes no availability blip, so it cannot inject spurious lifecycle + events into the next scenario. + + **Scope.** This operation resets flag state only. It MUST NOT be + expected to start a backend that is currently stopped — that is what + `/start` is for. A TCK therefore uses `/reset` only when the backend is + known to be running, and `/start` otherwise. The reference client tracks + this: `/stop` and `/restart` mark the backend as possibly-unreachable, so + the scenario that follows either of them is prepared with `/start`. + + **Fallback when not implemented.** A backend that does not implement this + operation MUST respond `404` or `501`. The TCK then falls back to + `POST /start?config={defaultConfig}`, which resets flag state at the cost + of a process restart. The fallback is detected once per suite and cached. + + Implementing `/reset` is RECOMMENDED for providers whose reconnect + behaviour makes the `/start` blip hard to distinguish from a real event. + responses: + "200": + description: Flag state restored to the baseline. + "404": + description: Not implemented; the TCK falls back to `/start`. + "501": + description: Not implemented; the TCK falls back to `/start`. + + /healthz: + get: + tags: [health] + operationId: health + summary: "[OPTIONAL] Readiness of the control API" + description: | + Reports whether the control API is ready to accept commands. + + **Fallback when not implemented.** Readiness defaults to "the control + port accepts a TCP connection", which the TCK establishes with a + Testcontainers listening-port wait strategy before the first scenario. A + `404` here is therefore not a failure, and the reference implementation + does not serve this path. + + Note this reports the health of the **control API**, not of the backend. + The backend is deliberately unhealthy during outage scenarios while the + control API must stay reachable — otherwise the TCK could not end the + outage. + responses: + "200": + description: Control API ready. + content: + application/json: + schema: + $ref: "#/components/schemas/Health" + "404": + description: Not implemented; readiness falls back to a TCP port check. + "503": + description: Control API not ready yet. + +components: + schemas: + + Health: + type: object + properties: + status: + type: string + enum: [ok] + description: Present and equal to `ok` when the control API is ready. + required: [status] + + Error: + type: object + properties: + message: + type: string + description: Human-readable explanation. Never interpreted by the TCK. + required: [message] diff --git a/tools/provider-tck/version.txt b/tools/provider-tck/version.txt new file mode 100644 index 000000000..8acdd82b7 --- /dev/null +++ b/tools/provider-tck/version.txt @@ -0,0 +1 @@ +0.0.1