diff --git a/Makefile b/Makefile index 3ee8b32487..13edf34fe2 100644 --- a/Makefile +++ b/Makefile @@ -153,6 +153,12 @@ endef # Create a target for each component $(foreach C, $(HELPER_BINARIES), $(eval $(call helper_target_template,$(C)))) +# Unlike the other devex/cmd helpers (built but left for the engineer to install/run by name), +# check-aws-marketplace-skew is meant to be run directly via `make` — build it and execute it in +# one step. Extra flags can be passed via ARGS, e.g. `make check-aws-marketplace-skew ARGS="--json"`. +check-aws-marketplace-skew: _build-helper-check-aws-marketplace-skew + @_output/linux/$(GOARCH)/check-aws-marketplace-skew $(ARGS) + define verify_e2e_target_template = .PHONY: $(1) $(1): _verify-e2e-$(1) diff --git a/devex/cmd/check-aws-marketplace-skew/README.md b/devex/cmd/check-aws-marketplace-skew/README.md new file mode 100644 index 0000000000..a3fe6836f7 --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/README.md @@ -0,0 +1,90 @@ +# check-aws-marketplace-skew + +A `make` target an engineer runs occasionally to check whether published AWS Marketplace RHCOS +AMIs have drifted out of MCO's boot-image skew band, so a stale or mismatched Marketplace image +can be caught before customers hit it. + +For each known Marketplace product code (OCP/OKE/OPP × x86_64/arm64, plus their EMEA x86_64 +variants — ROSA Classic is excluded, as it's being sunset), it fetches every published AMI and +checks whether **at least one** falls within an acceptable version band: + +- **Floor**: `RHCOSVersionBootImageSkewLimit`/`OCPVersionBootImageSkewLimit` + (`pkg/controller/common/constants.go`) as they stood ~4 months ago, reconstructed from + `openshift/machine-config-operator`'s GitHub history for `--branch` — not persisted state — so + Marketplace gets a grace period to catch up after MCO bumps its skew limit. +- **Ceiling**: the pinned RHCOS version for `--branch`, fetched live from `openshift/installer`'s + `data/data/coreos/coreos-rhel-10.json` on the matching branch. + +The check is existential, not singular: Marketplace can keep multiple AMI versions live at once, so +the current/default one being out of band doesn't necessarily mean there's no compliant option. + +## Usage + +```console +$ make check-aws-marketplace-skew +Skew-limit floor: RHCOS 9.2 (OCP 4.13.0) +Installer ceiling (x86_64): 10.2 +Installer ceiling (arm64): 10.2 + +PRODUCT PRODUCT ID RESULT MATCHED AMI DETAIL +OCP x86_64 59ead7de-2540-4653-a8b0-fa7926d5c845 PASS ami-0123456789abcdef0 9.6.20260210-0 +OKE x86_64 963b36c3-de6f-48ed-b802-2b38b2a2cdeb PASS ami-0fedcba9876543210 9.6.20260210-0 +... +``` + +Pass flags through with `ARGS`, e.g. `make check-aws-marketplace-skew ARGS="--json"`. + +Flags: + +- `--region` (default `us-east-1`): AWS region to query `DescribeImages` in. A single region is + sufficient — the version signal lives in the AMI `Name`/`Description` text, which is consistent + across every region a Marketplace listing replicates to. +- `--profile`: named AWS profile to use. Defaults to whatever the standard credential chain + resolves — if `aws` CLI commands already work for you, this tool will too. +- `--branch` (default `main`): the release branch to check, applied on both sides — the MCO + skew-limit floor is reconstructed from `openshift/machine-config-operator`'s history for this + branch, and the RHCOS ceiling is fetched from `openshift/installer`'s copy of the same branch + name. Both are fetched live from GitHub; no local checkout of either repo is needed, so this + works the same regardless of what `origin` points at locally (e.g. a personal fork that doesn't + mirror release branches). +- `--json`: emit a structured JSON report instead of a human-readable table. + +Exit code is non-zero if any product code fails its band check. + +## Credentials + +Requires the `aws` CLI to be installed and on `PATH` — this tool shells out to +`aws ec2 describe-images` rather than using the AWS Go SDK, so it has no AWS SDK dependency of its +own and just inherits whatever credentials already make `aws` CLI commands work for you +(environment variables, shared config/credentials file, SSO sessions). Use `--profile` to select a +named profile, same as `aws --profile`. + +The skew-limit floor also makes one `api.github.com` call per run (listing commits touching +`pkg/controller/common/constants.go`), which is unauthenticated by default and subject to GitHub's +60-requests/hour-per-IP limit — easy to hit on a shared NAT. Set `GITHUB_TOKEN` (the standard env +var honored by `gh`, GitHub Actions, etc.) to raise that to 5000/hour; no new flag needed. The +`raw.githubusercontent.com` fetches (installer ceiling, and the floor's file-at-commit lookup) +aren't subject to this same limit and don't need a token. + +## Known gaps + +- There are no AWS credentials in this repo's own CI, so `DescribeImages`/`CheckProduct` aren't + exercised end-to-end by `go test` — those tests use fixtures instead. Verified manually against + real Marketplace AMIs. +- Similarly, calls against the *real* GitHub endpoints aren't exercised by `go test` — this + package's tests run as part of the whole repo's presubmit suite, and a real network dependency + there would trade one devex tool's coverage for flakiness across every PR. `githubCommitsForPath` + itself (pagination, stop-on-short-page, 404 handling) *is* unit-tested against a local + `httptest.Server`, as are the pure parsing/selection functions (`parseGitHubCommitsJSON`, + `selectHistoricalCommit`, `parseSkewLimitConstants`, `parseStreamCeiling`) — only the thin + HTTP-call wrappers (`fetchGitHubCommitsPage`'s live request, `fetchRawGitHubFile`) are verified + manually against the real API instead. +- The skew-limit floor reconstruction fetches at most 500 commits (5 pages of 100 — GitHub's own + per-page cap) touching `pkg/controller/common/constants.go` on `--branch`, stopping early once a + page comes back short. Real history has 63 commits touching that file today, so this is + comfortable headroom, but if that count ever passes 500 for a branch, the "no commit predates the + grace window" fallback would pick the oldest of the *fetched* commits rather than the file's true + oldest revision. +- `openshift/installer`'s stream metadata filename is RHEL-major-version-specific + (`coreos-rhel-10.json`) — will need a manual update when RHEL 11 lands. Not building dynamic + discovery for this now. diff --git a/devex/cmd/check-aws-marketplace-skew/github.go b/devex/cmd/check-aws-marketplace-skew/github.go new file mode 100644 index 0000000000..408947cb47 --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/github.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" +) + +// githubRawURLTemplate points at raw.githubusercontent.com, which serves a file's content at any +// ref (branch, tag, or commit SHA) without needing a local checkout of the repo. +const githubRawURLTemplate = "https://raw.githubusercontent.com/%s/%s/%s/%s" + +// githubAPIBaseURL is a var (not a const) so tests can point it at an httptest.Server instead of +// the real GitHub API. +var githubAPIBaseURL = "https://api.github.com" + +// githubCommitsAPITemplate lists commits touching a path, most recent first. +const githubCommitsAPITemplate = "%s/repos/%s/%s/commits?sha=%s&path=%s&per_page=%d&page=%d" + +const ( + // githubCommitsPerPage is GitHub's own hard cap — the API rejects/clamps anything higher. + githubCommitsPerPage = 100 + // githubCommitsMaxPages bounds the total history considered to githubCommitsPerPage * + // githubCommitsMaxPages = 500 commits. Fetching stops as soon as a page comes back short + // (the true end of history for that path), so real MCO history (63 commits touching + // pkg/controller/common/constants.go today) still costs a single request — this cap only + // matters for branches/files with much deeper history than that. + githubCommitsMaxPages = 5 +) + +// RefNotFoundError indicates a GitHub API request returned HTTP 404. Path is set only when the +// request was for a specific file (raw.githubusercontent.com), where a 404 can mean either the +// ref or the path doesn't exist and the response body gives no way to tell which; for the commits +// API, a 404 unambiguously means the ref doesn't exist, so Path is left empty there. +type RefNotFoundError struct { + Owner, Repo, Ref, Path, URL string +} + +func (e *RefNotFoundError) Error() string { + if e.Path != "" { + return fmt.Sprintf("%s/%s: ref %q or path %q not found (%s returned 404)", e.Owner, e.Repo, e.Ref, e.Path, e.URL) + } + return fmt.Sprintf("%s/%s has no branch %q (%s returned 404)", e.Owner, e.Repo, e.Ref, e.URL) +} + +// fetchRawGitHubFile fetches path's content at ref from owner/repo over HTTPS, with no local +// checkout of that repo required. +func fetchRawGitHubFile(ctx context.Context, owner, repo, ref, path string) ([]byte, error) { + url := fmt.Sprintf(githubRawURLTemplate, owner, repo, ref, path) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, &RefNotFoundError{Owner: owner, Repo: repo, Ref: ref, Path: path, URL: url} + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("fetching %s returned HTTP %d: %s", url, resp.StatusCode, string(body)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body from %s: %w", url, err) + } + return body, nil +} + +// ghCommit is the subset of a GitHub API commit resource this tool needs. +type ghCommit struct { + SHA string + Date time.Time +} + +// githubCommitsForPath lists commits touching path on branch in owner/repo, most recent first, via +// the GitHub REST API rather than a local git checkout. Paginates up to githubCommitsMaxPages, +// stopping as soon as a page comes back short of githubCommitsPerPage (the true end of history). +func githubCommitsForPath(ctx context.Context, owner, repo, branch, path string) ([]ghCommit, error) { + var all []ghCommit + for page := 1; page <= githubCommitsMaxPages; page++ { + commits, err := fetchGitHubCommitsPage(ctx, owner, repo, branch, path, page) + if err != nil { + return nil, err + } + all = append(all, commits...) + if len(commits) < githubCommitsPerPage { + break + } + } + return all, nil +} + +// fetchGitHubCommitsPage fetches a single page of githubCommitsForPath's results. +func fetchGitHubCommitsPage(ctx context.Context, owner, repo, branch, path string, page int) ([]ghCommit, error) { + url := fmt.Sprintf(githubCommitsAPITemplate, githubAPIBaseURL, owner, repo, branch, path, githubCommitsPerPage, page) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + // api.github.com (unlike raw.githubusercontent.com) enforces a 60/hour unauthenticated rate + // limit per IP, shared across everything on that IP — cheap to blow through on a shared NAT. + // GITHUB_TOKEN is the standard env var convention (gh CLI, GitHub Actions, etc.); honoring it + // here raises the limit to 5000/hour with zero new flags for the common case of already having + // one set. + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", url, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body from %s: %w", url, err) + } + + if resp.StatusCode == http.StatusNotFound { + return nil, &RefNotFoundError{Owner: owner, Repo: repo, Ref: branch, URL: url} + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetching %s returned HTTP %d: %s", url, resp.StatusCode, string(body)) + } + + commits, err := parseGitHubCommitsJSON(body) + if err != nil { + return nil, fmt.Errorf("branch %s: %w", branch, err) + } + return commits, nil +} + +// parseGitHubCommitsJSON extracts SHA/committer-date pairs from a GitHub "list commits" API +// response, split out from githubCommitsForPath so it's testable against a fixture without a live +// network call. +func parseGitHubCommitsJSON(body []byte) ([]ghCommit, error) { + var raw []struct { + SHA string `json:"sha"` + Commit struct { + Committer struct { + Date time.Time `json:"date"` + } `json:"committer"` + } `json:"commit"` + } + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("failed to parse commits list: %w", err) + } + + commits := make([]ghCommit, 0, len(raw)) + for _, c := range raw { + commits = append(commits, ghCommit{SHA: c.SHA, Date: c.Commit.Committer.Date}) + } + return commits, nil +} diff --git a/devex/cmd/check-aws-marketplace-skew/github_test.go b/devex/cmd/check-aws-marketplace-skew/github_test.go new file mode 100644 index 0000000000..dfc36a85e8 --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/github_test.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sampleCommitsJSON mirrors the shape of a GitHub "list commits" API response, trimmed to the +// fields parseGitHubCommitsJSON reads. +const sampleCommitsJSON = `[ + { + "sha": "abc123", + "commit": { + "committer": { + "date": "2026-03-01T12:00:00Z" + } + } + }, + { + "sha": "def456", + "commit": { + "committer": { + "date": "2025-12-15T08:30:00Z" + } + } + } +]` + +func TestParseGitHubCommitsJSON(t *testing.T) { + commits, err := parseGitHubCommitsJSON([]byte(sampleCommitsJSON)) + require.NoError(t, err) + require.Len(t, commits, 2) + + assert.Equal(t, "abc123", commits[0].SHA) + assert.True(t, commits[0].Date.Equal(time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC))) + + assert.Equal(t, "def456", commits[1].SHA) + assert.True(t, commits[1].Date.Equal(time.Date(2025, 12, 15, 8, 30, 0, 0, time.UTC))) +} + +func TestParseGitHubCommitsJSON_empty(t *testing.T) { + commits, err := parseGitHubCommitsJSON([]byte(`[]`)) + require.NoError(t, err) + assert.Empty(t, commits) +} + +func TestParseGitHubCommitsJSON_malformed(t *testing.T) { + _, err := parseGitHubCommitsJSON([]byte("not json")) + require.Error(t, err) +} + +// withFakeGitHubAPI points githubAPIBaseURL at a local httptest.Server for the duration of the +// test — fully hermetic (localhost-only), unlike a live call to the real GitHub API. +func withFakeGitHubAPI(t *testing.T, handler http.HandlerFunc) { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + original := githubAPIBaseURL + githubAPIBaseURL = server.URL + t.Cleanup(func() { githubAPIBaseURL = original }) +} + +// fakeCommitsJSON renders shas as a GitHub "list commits" API response body. +func fakeCommitsJSON(t *testing.T, shas []string) []byte { + t.Helper() + type commit struct { + SHA string `json:"sha"` + Commit struct { + Committer struct { + Date time.Time `json:"date"` + } `json:"committer"` + } `json:"commit"` + } + commits := make([]commit, len(shas)) + for i, sha := range shas { + commits[i].SHA = sha + commits[i].Commit.Committer.Date = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(-time.Duration(i) * 24 * time.Hour) + } + body, err := json.Marshal(commits) + require.NoError(t, err) + return body +} + +func TestGithubCommitsForPath_paginatesUntilShortPage(t *testing.T) { + const total = 250 // spans 3 pages: 100, 100, 50 — proves it stops as soon as a page is short. + requestCount := 0 + withFakeGitHubAPI(t, func(w http.ResponseWriter, r *http.Request) { + requestCount++ + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + perPage, _ := strconv.Atoi(r.URL.Query().Get("per_page")) + require.Equal(t, githubCommitsPerPage, perPage) + + start := (page - 1) * perPage + end := start + perPage + if start > total { + start = total + } + if end > total { + end = total + } + shas := make([]string, 0, end-start) + for i := start; i < end; i++ { + shas = append(shas, fmt.Sprintf("sha-%d", i)) + } + _, err := w.Write(fakeCommitsJSON(t, shas)) + assert.NoError(t, err) + }) + + commits, err := githubCommitsForPath(context.Background(), "owner", "repo", "main", "path") + require.NoError(t, err) + assert.Len(t, commits, total) + assert.Equal(t, 3, requestCount) + assert.Equal(t, "sha-0", commits[0].SHA) + assert.Equal(t, fmt.Sprintf("sha-%d", total-1), commits[total-1].SHA) +} + +func TestGithubCommitsForPath_capsAtMaxPages(t *testing.T) { + requestCount := 0 + withFakeGitHubAPI(t, func(w http.ResponseWriter, r *http.Request) { + requestCount++ + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + perPage, _ := strconv.Atoi(r.URL.Query().Get("per_page")) + // The server has "infinite" history — always returns a full page — proving the client + // caps itself at githubCommitsMaxPages rather than looping forever. + shas := make([]string, perPage) + for i := range shas { + shas[i] = fmt.Sprintf("page%d-%d", page, i) + } + _, err := w.Write(fakeCommitsJSON(t, shas)) + assert.NoError(t, err) + }) + + commits, err := githubCommitsForPath(context.Background(), "owner", "repo", "main", "path") + require.NoError(t, err) + assert.Len(t, commits, githubCommitsPerPage*githubCommitsMaxPages) + assert.Equal(t, githubCommitsMaxPages, requestCount) +} + +func TestGithubCommitsForPath_branchNotFound(t *testing.T) { + withFakeGitHubAPI(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, err := w.Write([]byte(`{"message": "Not Found"}`)) + assert.NoError(t, err) + }) + + _, err := githubCommitsForPath(context.Background(), "owner", "repo", "nonexistent-branch", "path") + require.Error(t, err) + var notFound *RefNotFoundError + require.ErrorAs(t, err, ¬Found) + assert.Equal(t, "nonexistent-branch", notFound.Ref) +} diff --git a/devex/cmd/check-aws-marketplace-skew/installerceiling.go b/devex/cmd/check-aws-marketplace-skew/installerceiling.go new file mode 100644 index 0000000000..c471317a4b --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/installerceiling.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + coreosstream "github.com/coreos/stream-metadata-go/stream" +) + +// installerStreamPath is openshift/installer's pinned RHCOS stream metadata — the same file the +// installer itself uses to select RHCOS for a release. +const installerStreamPath = "data/data/coreos/coreos-rhel-10.json" + +// marketplaceArchToStreamArch maps a Marketplace product's arch label to the stream metadata's +// architecture key. +var marketplaceArchToStreamArch = map[string]string{ + "x86_64": "x86_64", + "arm64": "aarch64", +} + +// FetchInstallerCeilings fetches openshift/installer's pinned RHCOS stream metadata for branch +// once and returns the "release" field's major.minor token (e.g. "10.2") for every supported +// Marketplace architecture, keyed by Marketplace arch label — the ceiling of the acceptable skew +// band, since a Marketplace AMI newer than this indicates Marketplace is serving the wrong image +// for this branch. Both archs live in the same stream-metadata file, so this fetches it once +// rather than once per arch. +func FetchInstallerCeilings(ctx context.Context, branch string) (map[string]string, error) { + body, err := fetchRawGitHubFile(ctx, "openshift", "installer", branch, installerStreamPath) + if err != nil { + return nil, err + } + + ceilings := make(map[string]string, len(marketplaceArchToStreamArch)) + for marketplaceArch, streamArch := range marketplaceArchToStreamArch { + token, _, err := parseStreamCeiling(body, streamArch) + if err != nil { + return nil, fmt.Errorf("branch %s arch %s: %w", branch, marketplaceArch, err) + } + ceilings[marketplaceArch] = token + } + return ceilings, nil +} + +// parseStreamCeiling extracts the aws artifact's release token from raw stream metadata JSON for +// streamArch, split out from FetchInstallerCeilings so it can be tested against a fixture without a +// live network call. +func parseStreamCeiling(body []byte, streamArch string) (token, fullRelease string, err error) { + var streamData coreosstream.Stream + if err := json.Unmarshal(body, &streamData); err != nil { + return "", "", fmt.Errorf("failed to parse stream metadata: %w", err) + } + + arch, err := streamData.GetArchitecture(streamArch) + if err != nil { + return "", "", err + } + + awsArtifact, ok := arch.Artifacts["aws"] + if !ok { + return "", "", fmt.Errorf("stream metadata has no aws artifact for %s", streamArch) + } + + fullRelease = awsArtifact.Release + token, err = releaseToken(fullRelease) + if err != nil { + return "", "", err + } + return token, fullRelease, nil +} + +// releaseToken derives the major.minor token from a full RHCOS release string, e.g. +// "10.2.20260423-0" -> "10.2". +func releaseToken(release string) (string, error) { + parts := strings.SplitN(release, ".", 3) + if len(parts) < 2 { + return "", fmt.Errorf("unexpected RHCOS release string format: %q", release) + } + return parts[0] + "." + parts[1], nil +} diff --git a/devex/cmd/check-aws-marketplace-skew/installerceiling_test.go b/devex/cmd/check-aws-marketplace-skew/installerceiling_test.go new file mode 100644 index 0000000000..9f71dd98ad --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/installerceiling_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sampleStreamJSON mirrors the real shape of openshift/installer's coreos-rhel-10.json. +const sampleStreamJSON = `{ + "stream": "rhcos-4.22", + "architectures": { + "aarch64": { + "artifacts": { + "aws": { + "release": "10.2.20260423-0" + } + } + }, + "x86_64": { + "artifacts": { + "aws": { + "release": "10.2.20260423-0" + } + } + } + } +}` + +func TestParseStreamCeiling(t *testing.T) { + cases := []struct { + name string + body string + streamArch string + wantToken string + wantFullVer string + expectError bool + }{ + { + name: "x86_64", + body: sampleStreamJSON, + streamArch: "x86_64", + wantToken: "10.2", + wantFullVer: "10.2.20260423-0", + }, + { + name: "aarch64", + body: sampleStreamJSON, + streamArch: "aarch64", + wantToken: "10.2", + wantFullVer: "10.2.20260423-0", + }, + { + name: "unknown arch", + body: sampleStreamJSON, + streamArch: "s390x", + expectError: true, + }, + { + name: "malformed json", + body: "not json", + streamArch: "x86_64", + expectError: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + token, fullRelease, err := parseStreamCeiling([]byte(tc.body), tc.streamArch) + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantToken, token) + assert.Equal(t, tc.wantFullVer, fullRelease) + }) + } +} + +func TestReleaseToken(t *testing.T) { + cases := []struct { + name string + release string + want string + expectError bool + }{ + {"standard release string", "10.2.20260423-0", "10.2", false}, + {"double digit minor", "9.10.20260210-0", "9.10", false}, + {"two segment string", "9.6", "9.6", false}, + {"empty string", "", "", true}, + {"single segment", "9", "", true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := releaseToken(tc.release) + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/devex/cmd/check-aws-marketplace-skew/main.go b/devex/cmd/check-aws-marketplace-skew/main.go new file mode 100644 index 0000000000..d8f402a089 --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" + "k8s.io/component-base/cli" +) + +// defaultBranch is the branch queried on both the MCO and openshift/installer side when +// --branch isn't set. +const defaultBranch = "main" + +// runTimeout bounds the whole check: a handful of paginated GitHub API calls plus one `aws` +// CLI subprocess per product spec. Generous headroom over the happy-path cost so a hung network +// call or subprocess can't block forever, without needing per-call timeouts of its own. +const runTimeout = 2 * time.Minute + +func main() { + var ( + region string + profile string + branch string + jsonOut bool + ) + + rootCmd := &cobra.Command{ + Use: "check-aws-marketplace-skew", + Short: "Checks published AWS Marketplace RHCOS AMIs against the MCO boot-image skew band.", + RunE: func(_ *cobra.Command, _ []string) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(ctx, runTimeout) + defer cancel() + return run(ctx, region, profile, branch, jsonOut) + }, + } + + rootCmd.PersistentFlags().StringVar(®ion, "region", "us-east-1", "AWS region to query DescribeImages against. A single region is sufficient: the version signal lives in the AMI Name/Description text, which is consistent across every region a Marketplace listing replicates to.") + rootCmd.PersistentFlags().StringVar(&profile, "profile", "", "named AWS profile to use (default: whatever the default credential chain resolves, same as the aws CLI)") + rootCmd.PersistentFlags().StringVar(&branch, "branch", defaultBranch, "release branch to check, applied to both the MCO skew-limit history and the openshift/installer RHCOS ceiling — both fetched live from GitHub, no local checkout of either repo needed") + rootCmd.PersistentFlags().BoolVar(&jsonOut, "json", false, "emit structured JSON instead of a human-readable table") + + os.Exit(cli.Run(rootCmd)) +} + +func run(ctx context.Context, region, profile, branch string, jsonOut bool) error { + floor, err := HistoricalSkewLimits(ctx, branch, time.Now()) + if err != nil { + return err + } + + ceilings, err := FetchInstallerCeilings(ctx, branch) + if err != nil { + return err + } + + report := Report{Floor: floor, Ceiling: ceilings} + for _, product := range allProductSpecs() { + result, err := CheckProduct(ctx, region, profile, product, floor.RHCOS, ceilings[product.Arch]) + if err != nil { + return fmt.Errorf("checking product %s (%s): %w", product.Name, product.ID, err) + } + report.Results = append(report.Results, result) + } + + if jsonOut { + if err := report.WriteJSON(os.Stdout); err != nil { + return err + } + } else if err := report.WriteTable(os.Stdout); err != nil { + return err + } + + if report.AnyFailed() { + failed := 0 + for _, res := range report.Results { + if !res.Pass { + failed++ + } + } + return fmt.Errorf("%d of %d product codes failed the skew band check", failed, len(report.Results)) + } + return nil +} diff --git a/devex/cmd/check-aws-marketplace-skew/marketplace.go b/devex/cmd/check-aws-marketplace-skew/marketplace.go new file mode 100644 index 0000000000..d0e33d2e9b --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/marketplace.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" + + bootimagemarketplace "github.com/openshift/machine-config-operator/pkg/controller/bootimage/marketplace" +) + +// CLIImage is the subset of `aws ec2 describe-images --output json` fields this tool needs. The +// AWS CLI's JSON output uses the same field names as the API/Go SDK (ImageId, Name, Description, +// CreationDate), so no translation layer is needed beyond ignoring the fields we don't use. +type CLIImage struct { + ImageID string `json:"ImageId"` + Name string + Description string +} + +type describeImagesOutput struct { + Images []CLIImage +} + +// DescribeMarketplaceAMIs returns every published Marketplace AMI whose name contains productID, +// by shelling out to the aws CLI rather than depending on the AWS Go SDK. This keeps the tool free +// of any AWS SDK dependency — it just needs whatever credentials already make `aws` CLI commands +// work for the engineer running it. +func DescribeMarketplaceAMIs(ctx context.Context, region, profile, productID string) ([]CLIImage, error) { + args := []string{ + "ec2", "describe-images", + "--owners", "aws-marketplace", + "--filters", "Name=name,Values=*" + productID + "*", + "--region", region, + "--output", "json", + } + if profile != "" { + args = append(args, "--profile", profile) + } + + cmd := exec.CommandContext(ctx, "aws", args...) + out, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("aws %v failed: %w: %s", args, err, exitErr.Stderr) + } + return nil, fmt.Errorf("failed to run aws CLI (is it installed and on PATH?): %w", err) + } + + var parsed describeImagesOutput + if err := json.Unmarshal(out, &parsed); err != nil { + return nil, fmt.Errorf("failed to parse aws ec2 describe-images output: %w", err) + } + return parsed.Images, nil +} + +// tokenInBand reports whether token falls within [floor, ceiling], inclusive. +func tokenInBand(token, floor, ceiling string) bool { + return bootimagemarketplace.CmpVersionToken(token, floor) >= 0 && bootimagemarketplace.CmpVersionToken(token, ceiling) <= 0 +} + +// AMIMatch describes the Marketplace AMI, if any, that satisfied a product's band check. +type AMIMatch struct { + ImageID, Name, Description, Version, Token string +} + +// ProductResult is the pass/fail outcome of the band check for a single Marketplace product. +type ProductResult struct { + ProductID, ProductName string + Pass bool + MatchedAMI *AMIMatch + CandidateCount int // how many AMIs were found in total, for FAIL diagnostics + Reason string // set on FAIL or error +} + +// CheckProduct is existential, not singular: it enumerates every published Marketplace AMI for +// product and passes if at least one falls within [floor, ceiling]. Marketplace may keep multiple +// AMI versions live at once, so the default/latest one being out of band doesn't mean customers +// have no compliant option. +func CheckProduct(ctx context.Context, region, profile string, product ProductSpec, floor, ceiling string) (ProductResult, error) { + result := ProductResult{ProductID: product.ID, ProductName: product.Name} + + images, err := DescribeMarketplaceAMIs(ctx, region, profile, product.ID) + if err != nil { + return ProductResult{}, err + } + result.CandidateCount = len(images) + + var best *AMIMatch + for _, img := range images { + fullVersion, token, ok := bootimagemarketplace.ExtractVersionFromDescription(img.Description) + if !ok { + continue + } + if !tokenInBand(token, floor, ceiling) { + continue + } + if best == nil || bootimagemarketplace.CmpVersionToken(token, best.Token) > 0 { + best = &AMIMatch{ + ImageID: img.ImageID, + Name: img.Name, + Description: img.Description, + Version: fullVersion, + Token: token, + } + } + } + + if best == nil { + result.Pass = false + result.Reason = fmt.Sprintf("no published AMI in band [%s, %s] out of %d candidate(s)", floor, ceiling, result.CandidateCount) + return result, nil + } + + result.Pass = true + result.MatchedAMI = best + return result, nil +} diff --git a/devex/cmd/check-aws-marketplace-skew/marketplace_test.go b/devex/cmd/check-aws-marketplace-skew/marketplace_test.go new file mode 100644 index 0000000000..6568e2dd0b --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/marketplace_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTokenInBand(t *testing.T) { + cases := []struct { + name string + token, floor, ceiling string + expected bool + }{ + {"within band", "9.6", "9.2", "10.2", true}, + {"equal to floor", "9.2", "9.2", "10.2", true}, + {"equal to ceiling", "10.2", "9.2", "10.2", true}, + {"below floor", "9.1", "9.2", "10.2", false}, + {"above ceiling", "10.3", "9.2", "10.2", false}, + {"floor equals ceiling, exact match", "9.2", "9.2", "9.2", true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, tokenInBand(tc.token, tc.floor, tc.ceiling)) + }) + } +} diff --git a/devex/cmd/check-aws-marketplace-skew/products.go b/devex/cmd/check-aws-marketplace-skew/products.go new file mode 100644 index 0000000000..b74afb585d --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/products.go @@ -0,0 +1,56 @@ +package main + +import ( + "sort" + "strings" + + bootimagemarketplace "github.com/openshift/machine-config-operator/pkg/controller/bootimage/marketplace" +) + +// ProductSpec is a Marketplace product entry paired with the architecture whose installer +// ceiling it should be checked against. +type ProductSpec struct { + ID, Name, Arch string +} + +// archSortWeight orders x86_64 before arm64 in report output; alphabetical order would put arm64 +// first ('a' < 'x'), which reads backwards from how these are conventionally listed. +func archSortWeight(arch string) int { + if arch == "arm64" { + return 1 + } + return 0 +} + +// allProductSpecs derives arch for each shared-package product entry by inspecting its name. +// Every entry encodes its architecture in the name. Results are ordered by arch, then region +// (standard before EMEA), then name, so the report table reads as grouped sections instead of +// shuffled by the underlying (arbitrary) product UUID. +// +// ROSA Classic is excluded: it's being sunset, so its Marketplace image freshness is no longer +// worth tracking here. marketplace.ROSAProductID/Products["ROSA"] stay in the shared package +// as-is — the boot image controller still needs them for AMI-kind detection on existing clusters. +func allProductSpecs() []ProductSpec { + specs := make([]ProductSpec, 0, len(bootimagemarketplace.Products)) + for id, name := range bootimagemarketplace.Products { + if id == bootimagemarketplace.ROSAProductID { + continue + } + arch := "x86_64" + if strings.Contains(name, "arm64") { + arch = "arm64" + } + specs = append(specs, ProductSpec{ID: id, Name: name, Arch: arch}) + } + sort.Slice(specs, func(i, j int) bool { + a, b := specs[i], specs[j] + if wa, wb := archSortWeight(a.Arch), archSortWeight(b.Arch); wa != wb { + return wa < wb + } + if aEMEA, bEMEA := strings.Contains(a.Name, "EMEA"), strings.Contains(b.Name, "EMEA"); aEMEA != bEMEA { + return !aEMEA + } + return a.Name < b.Name + }) + return specs +} diff --git a/devex/cmd/check-aws-marketplace-skew/report.go b/devex/cmd/check-aws-marketplace-skew/report.go new file mode 100644 index 0000000000..23164a5e64 --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/report.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "text/tabwriter" +) + +// Report is the outcome of a check-aws-marketplace-skew run, across all Marketplace products checked. +type Report struct { + Floor SkewLimits `json:"floor"` + Ceiling map[string]string `json:"ceiling"` // arch -> token + Results []ProductResult `json:"results"` +} + +// AnyFailed reports whether any product failed its band check. +func (r Report) AnyFailed() bool { + for _, res := range r.Results { + if !res.Pass { + return true + } + } + return false +} + +// WriteTable renders a human-readable summary table. +func (r Report) WriteTable(w io.Writer) error { + if _, err := fmt.Fprintf(w, "Skew-limit floor: RHCOS %s (OCP %s)\n", r.Floor.RHCOS, r.Floor.OCP); err != nil { + return err + } + for arch, token := range r.Ceiling { + if _, err := fmt.Fprintf(w, "Installer ceiling (%s): %s\n", arch, token); err != nil { + return err + } + } + if _, err := fmt.Fprintln(w); err != nil { + return err + } + + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + if _, err := fmt.Fprintln(tw, "PRODUCT\tPRODUCT ID\tRESULT\tMATCHED AMI\tDETAIL"); err != nil { + return err + } + for _, res := range r.Results { + result := "PASS" + if !res.Pass { + result = "FAIL" + } + matched := "" + detail := res.Reason + if res.MatchedAMI != nil { + matched = res.MatchedAMI.ImageID + detail = res.MatchedAMI.Version + } + if _, err := fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", res.ProductName, res.ProductID, result, matched, detail); err != nil { + return err + } + } + return tw.Flush() +} + +// WriteJSON renders the report as structured JSON, for a future CI wrapper to consume when +// deciding whether to file/update a Jira bug. +func (r Report) WriteJSON(w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(r) +} diff --git a/devex/cmd/check-aws-marketplace-skew/skewlimit.go b/devex/cmd/check-aws-marketplace-skew/skewlimit.go new file mode 100644 index 0000000000..d0abed223f --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/skewlimit.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "time" + + "k8s.io/klog/v2" +) + +// SkewLimits holds the reconstructed value of MCO's boot-image skew-limit constants at a point in time. +type SkewLimits struct { + RHCOS string // e.g. "9.2" — the operative floor for AMI-token comparison + OCP string // e.g. "4.13.0" — surfaced for context/output only +} + +const ( + // graceMonths gives Marketplace publishers time to catch up after MCO bumps its skew limit. + graceMonths = 4 + // skewLimitsPath is relative to the MCO repo root. + skewLimitsPath = "pkg/controller/common/constants.go" +) + +// HistoricalSkewLimits reconstructs the RHCOSVersionBootImageSkewLimit/OCPVersionBootImageSkewLimit +// constants (pkg/controller/common/constants.go) as they stood graceMonths before asOf on branch, by +// querying openshift/machine-config-operator's GitHub history rather than a local checkout. This +// anchors the grace period to when the constants actually held a given value, so a double-bump +// within the window can't reset the clock the way anchoring to "time since last bump" would. +func HistoricalSkewLimits(ctx context.Context, branch string, asOf time.Time) (SkewLimits, error) { + commits, err := githubCommitsForPath(ctx, "openshift", "machine-config-operator", branch, skewLimitsPath) + if err != nil { + return SkewLimits{}, err + } + + cutoff := asOf.AddDate(0, -graceMonths, 0) + rev, usedFallback, err := selectHistoricalCommit(commits, cutoff) + if err != nil { + return SkewLimits{}, err + } + + src, err := fetchRawGitHubFile(ctx, "openshift", "machine-config-operator", rev, skewLimitsPath) + if err != nil { + return SkewLimits{}, err + } + + limits, err := parseSkewLimitConstants(string(src)) + if err != nil { + return SkewLimits{}, fmt.Errorf("skew-limit constants not present in %s as of commit %s (cutoff %s): %w", skewLimitsPath, rev, cutoff.Format(time.RFC3339), err) + } + + if usedFallback { + klog.Warningf("no commit predates the %d-month grace window; using the oldest available value of the skew-limit constants (commit %s)", graceMonths, rev) + } + + return limits, nil +} + +// selectHistoricalCommit picks the commit whose state was in effect as of cutoff from commits +// (assumed newest-first, as returned by the GitHub API): the most recent commit older than cutoff, +// or — if none predates the window (the file/branch is younger than graceMonths) — the oldest +// commit available, since there's no earlier data to grant a grace period against. +func selectHistoricalCommit(commits []ghCommit, cutoff time.Time) (sha string, usedFallback bool, err error) { + if len(commits) == 0 { + return "", false, fmt.Errorf("no commit history found for %s", skewLimitsPath) + } + for _, c := range commits { + if c.Date.Before(cutoff) { + return c.SHA, false, nil + } + } + return commits[len(commits)-1].SHA, true, nil +} + +var ( + rhcosSkewLimitRe = regexp.MustCompile(`RHCOSVersionBootImageSkewLimit\s*=\s*"([^"]+)"`) + ocpSkewLimitRe = regexp.MustCompile(`OCPVersionBootImageSkewLimit\s*=\s*"([^"]+)"`) +) + +// parseSkewLimitConstants extracts the two skew-limit string-literal constants from a copy of +// constants.go's source text. Deliberately does not build/vet the historical revision — that's +// slow, fragile (an old revision may not compile standalone against current vendor state), and +// unnecessary for extracting two string literals. +func parseSkewLimitConstants(src string) (SkewLimits, error) { + rhcosMatch := rhcosSkewLimitRe.FindStringSubmatch(src) + ocpMatch := ocpSkewLimitRe.FindStringSubmatch(src) + if rhcosMatch == nil || ocpMatch == nil { + return SkewLimits{}, fmt.Errorf("could not find both RHCOSVersionBootImageSkewLimit and OCPVersionBootImageSkewLimit constants") + } + return SkewLimits{RHCOS: rhcosMatch[1], OCP: ocpMatch[1]}, nil +} diff --git a/devex/cmd/check-aws-marketplace-skew/skewlimit_test.go b/devex/cmd/check-aws-marketplace-skew/skewlimit_test.go new file mode 100644 index 0000000000..3dc6b9bcae --- /dev/null +++ b/devex/cmd/check-aws-marketplace-skew/skewlimit_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSelectHistoricalCommit(t *testing.T) { + date := func(s string) time.Time { + t.Helper() + d, err := time.Parse(time.RFC3339, s) + require.NoError(t, err) + return d + } + + t.Run("double bump within grace window returns the value in effect at cutoff, not the latest bump", func(t *testing.T) { + // Bump B lands just before the cutoff; bump C lands just after it, inside the window. If + // the grace period were naively anchored to "time since the last bump" (bump C, ~3 months + // ago), it would look like there's still time left on the clock. Reconstructing the actual + // value as of the cutoff sidesteps that: it should return bump B, what was genuinely in + // effect 4 months ago, regardless of the later bump C. Commits are newest-first, matching + // the GitHub API's default order. + commits := []ghCommit{ + {SHA: "bumpC", Date: date("2026-03-01T00:00:00Z")}, // after cutoff, within the window + {SHA: "bumpB", Date: date("2025-12-15T00:00:00Z")}, // before cutoff + {SHA: "bumpA", Date: date("2025-01-01T00:00:00Z")}, // long-standing baseline + } + cutoff := date("2026-02-01T00:00:00Z") + + sha, usedFallback, err := selectHistoricalCommit(commits, cutoff) + require.NoError(t, err) + assert.Equal(t, "bumpB", sha) + assert.False(t, usedFallback) + }) + + t.Run("no commit predates the window falls back to the oldest available", func(t *testing.T) { + commits := []ghCommit{ + {SHA: "newer", Date: date("2026-05-01T00:00:00Z")}, + {SHA: "oldest", Date: date("2026-04-15T00:00:00Z")}, + } + cutoff := date("2026-01-01T00:00:00Z") + + sha, usedFallback, err := selectHistoricalCommit(commits, cutoff) + require.NoError(t, err) + assert.Equal(t, "oldest", sha) + assert.True(t, usedFallback) + }) + + t.Run("no commit history is an error", func(t *testing.T) { + _, _, err := selectHistoricalCommit(nil, date("2026-01-01T00:00:00Z")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no commit history found") + }) +} + +func TestParseSkewLimitConstants(t *testing.T) { + t.Run("extracts both constants", func(t *testing.T) { + src := "package common\n\nconst (\n\tRHCOSVersionBootImageSkewLimit = \"9.2\"\n\tOCPVersionBootImageSkewLimit = \"4.13.0\"\n)\n" + limits, err := parseSkewLimitConstants(src) + require.NoError(t, err) + assert.Equal(t, "9.2", limits.RHCOS) + assert.Equal(t, "4.13.0", limits.OCP) + }) + + t.Run("missing constants is an error", func(t *testing.T) { + _, err := parseSkewLimitConstants("package common\n") + require.Error(t, err) + assert.Contains(t, err.Error(), "could not find both") + }) +} diff --git a/pkg/controller/bootimage/aws_helpers.go b/pkg/controller/bootimage/aws_helpers.go index b69adfb1a5..c497507f15 100644 --- a/pkg/controller/bootimage/aws_helpers.go +++ b/pkg/controller/bootimage/aws_helpers.go @@ -3,9 +3,7 @@ package bootimage import ( "context" "fmt" - "regexp" "sort" - "strconv" "strings" "sync" @@ -21,6 +19,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "github.com/openshift/machine-config-operator/pkg/controller/bootimage/marketplace" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" ) @@ -29,8 +28,6 @@ const ( awsMarketplaceOwnerID = "679593333241" // awsRHCOSOwnerID is the Red Hat AWS account that owns standard RHCOS AMIs. awsRHCOSOwnerID = "531415883065" - // rosaProductID is the marketplace product ID for ROSA Classic. - rosaProductID = "34850061-abaf-402d-92df-94325c9e947f" // awsMarketplaceOwnerAlias is the owner alias used in DescribeImages filters for marketplace AMIs. awsMarketplaceOwnerAlias = "aws-marketplace" // awsCredentialsSecretName is the secret in openshift-machine-api provisioned by @@ -38,34 +35,6 @@ const ( awsCredentialsSecretName = "aws-cloud-credentials" ) -// marketplaceProductNames maps the AWS Marketplace product IDs to human-readable variant names. -// These IDs are stable — they are tied to marketplace listings and will not change. -var marketplaceProductNames = map[string]string{ - // x86_64 - "59ead7de-2540-4653-a8b0-fa7926d5c845": "OCP x86_64", - "963b36c3-de6f-48ed-b802-2b38b2a2cdeb": "OKE x86_64", - "f5da01a6-d046-487c-9072-42fe53b1cad4": "OPP x86_64", - // arm64 - "abc249f8-7440-45f7-a4b1-c026baff64c1": "OCP arm64", - "d2d3ebcd-c1ca-43d8-bf0a-530433200f35": "OKE arm64", - "be6d3e94-c8dc-4a3e-9218-4b449b11f06f": "OPP arm64", - // x86_64 EMEA - "962791c7-3ae5-46d1-ba62-c7a5ebac54fd": "OCP EMEA x86_64", - "7026c8d7-392c-4010-b93c-f93f7bc5495f": "OKE EMEA x86_64", - "628c9df3-0254-4f91-bc1f-8619d1b8eaa8": "OPP EMEA x86_64", - // ROSA - rosaProductID: "ROSA", -} - -// productName returns the human-readable variant name for a marketplace product ID, -// falling back to the product ID itself if it is not in the map. -func productName(productID string) string { - if name, ok := marketplaceProductNames[productID]; ok { - return name - } - return productID -} - // amiKind classifies a RHCOS AMI by its origin. type amiKind int @@ -173,11 +142,11 @@ func detectAMIKind(image *ec2types.Image) (amiKind, string) { case awsRHCOSOwnerID: return amiKindStandard, "" case awsMarketplaceOwnerID: - productID := extractProductID(aws.ToString(image.Name)) + productID := marketplace.ExtractProductID(aws.ToString(image.Name)) switch productID { case "": return amiKindUnknown, "" - case rosaProductID: + case marketplace.ROSAProductID: return amiKindROSA, productID default: return amiKindMarketplace, productID @@ -187,25 +156,6 @@ func detectAMIKind(image *ec2types.Image) (amiKind, string) { } } -var productIDRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) - -// extractProductID returns the trailing UUID-format product ID from a marketplace AMI name, e.g.: -// -// RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-{product-id} -// -// Returns an empty string if no valid product ID is found. -func extractProductID(name string) string { - parts := strings.Split(name, "-") - if len(parts) < 5 { - return "" - } - candidate := strings.Join(parts[len(parts)-5:], "-") - if productIDRegex.MatchString(candidate) { - return candidate - } - return "" -} - // marketplaceVersionToken derives the version token used in marketplace AMI descriptions // from the RHCOS release string in the stream configmap. // @@ -247,65 +197,6 @@ func resolveMarketplaceAMI(ctx context.Context, client *ec2.Client, streamData * return findMarketplaceAMI(ctx, client, productID, versionToken, machineSetName) } -// descriptionVersionRe matches the full RHCOS release string embedded in marketplace AMI descriptions. -// The full match is the API-valid release string; group 1 is the N.M token for version comparison. -// Both description formats in use embed the full release string inline: -// - RHEL marketplace: "RHEL CoreOS 9.6 9.6.20260210-0 x86_64" → "9.6.20260210-0" / "9.6" -// - ROSA: "rhcos-9.6.20250701-0-x86_64" → "9.6.20250701-0" / "9.6" -var descriptionVersionRe = regexp.MustCompile(`(\d+\.\d+)\.(?:[0-9]{8}|[0-9]{12})-\d+`) - -// extractVersionFromDescription parses the RHCOS release string from a marketplace AMI description. -// Returns the full release string (e.g. "9.6.20260210-0") suitable for ClusterBootImageAutomatic.RHCOSVersion, -// the N.M token (e.g. "9.6") for version comparison, and whether parsing succeeded. -func extractVersionFromDescription(description string) (fullVersion, token string, ok bool) { - m := descriptionVersionRe.FindStringSubmatch(description) - if m == nil { - return "", "", false - } - return m[0], m[1], true -} - -// cmpRHCOSVersion compares two full RHCOS release strings (e.g. "9.6.20260210-0") by their -// major.minor version only. Returns negative if a < b, zero if equal, positive if a > b. -func cmpRHCOSVersion(a, b string) int { - tokenOf := func(v string) string { - p := strings.SplitN(v, ".", 3) - if len(p) < 2 { - return v - } - return p[0] + "." + p[1] - } - return cmpVersionToken(tokenOf(a), tokenOf(b)) -} - -// cmpVersionToken compares two "major.minor" version tokens. -// Returns negative if a < b, zero if equal, positive if a > b. -func cmpVersionToken(a, b string) int { - parse := func(s string) (int, int) { - parts := strings.SplitN(s, ".", 2) - if len(parts) != 2 { - return 0, 0 - } - major, _ := strconv.Atoi(parts[0]) - minor, _ := strconv.Atoi(parts[1]) - return major, minor - } - aMaj, aMin := parse(a) - bMaj, bMin := parse(b) - if aMaj != bMaj { - return aMaj - bMaj - } - return aMin - bMin -} - -// isPreRHELAlignedToken reports whether a version token uses the pre-4.19 OCP-based RHCOS -// versioning scheme (e.g. "418.94") rather than the RHEL-aligned scheme (e.g. "9.6"). -// Pre-RHEL-aligned tokens have a major component > 100 (encoding the OCP major version * 100 + minor). -func isPreRHELAlignedToken(token string) bool { - major, _ := strconv.Atoi(strings.SplitN(token, ".", 2)[0]) - return major > 100 -} - // findMarketplaceAMI returns the AMI ID and RHCOS version of the best marketplace AMI for the // given product ID and version token. It fetches all AMIs for the product ID, discards any whose // description version exceeds the target, then returns the newest AMI at the highest version ≤ target. @@ -336,13 +227,13 @@ func findMarketplaceAMI(ctx context.Context, client *ec2.Client, productID, vers var matches []candidate var sawPreRHELAligned bool for _, img := range out.Images { - fullVersion, token, ok := extractVersionFromDescription(aws.ToString(img.Description)) + fullVersion, token, ok := marketplace.ExtractVersionFromDescription(aws.ToString(img.Description)) if !ok { continue } - if cmpVersionToken(token, versionToken) <= 0 { + if marketplace.CmpVersionToken(token, versionToken) <= 0 { matches = append(matches, candidate{img, token, fullVersion}) - } else if isPreRHELAlignedToken(token) { + } else if marketplace.IsPreRHELAlignedToken(token) { sawPreRHELAligned = true } } @@ -361,7 +252,7 @@ func findMarketplaceAMI(ctx context.Context, client *ec2.Client, productID, vers // Prefer the highest version not exceeding the target; break ties by newest CreationDate. sort.Slice(matches, func(i, j int) bool { - if cmp := cmpVersionToken(matches[i].token, matches[j].token); cmp != 0 { + if cmp := marketplace.CmpVersionToken(matches[i].token, matches[j].token); cmp != 0 { return cmp > 0 } return aws.ToString(matches[i].image.CreationDate) > aws.ToString(matches[j].image.CreationDate) diff --git a/pkg/controller/bootimage/aws_helpers_test.go b/pkg/controller/bootimage/aws_helpers_test.go index 9daf903b15..330869b1a1 100644 --- a/pkg/controller/bootimage/aws_helpers_test.go +++ b/pkg/controller/bootimage/aws_helpers_test.go @@ -17,56 +17,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" ) -func TestExtractProductID(t *testing.T) { - cases := []struct { - name string - amiName string - expected string - }{ - { - name: "OCP x86_64 AMI", - amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-59ead7de-2540-4653-a8b0-fa7926d5c845", - expected: "59ead7de-2540-4653-a8b0-fa7926d5c845", - }, - { - name: "OCP arm64 AMI", - amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-aarch64-0-abc249f8-7440-45f7-a4b1-c026baff64c1", - expected: "abc249f8-7440-45f7-a4b1-c026baff64c1", - }, - { - name: "ROSA AMI", - amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-34850061-abaf-402d-92df-94325c9e947f", - expected: "34850061-abaf-402d-92df-94325c9e947f", - }, - { - name: "standard RHCOS AMI has no product ID", - amiName: "rhcos-419-94-202504151514-0-x86_64", - expected: "", - }, - { - name: "name too short", - amiName: "short", - expected: "", - }, - { - name: "trailing segment not a UUID", - amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-notauuid", - expected: "", - }, - { - name: "empty name", - amiName: "", - expected: "", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expected, extractProductID(tc.amiName)) - }) - } -} - func TestMarketplaceVersionToken(t *testing.T) { cases := []struct { name string @@ -114,141 +64,6 @@ func TestMarketplaceVersionToken(t *testing.T) { } } -func TestExtractVersionFromDescription(t *testing.T) { - cases := []struct { - name string - description string - fullVersion string - token string - ok bool - }{ - { - name: "RHEL marketplace format x86_64", - description: "RHEL CoreOS 9.6 9.6.20260210-0 x86_64", - fullVersion: "9.6.20260210-0", - token: "9.6", - ok: true, - }, - { - name: "RHEL marketplace format aarch64", - description: "RHEL CoreOS 9.6 9.6.20260210-0 aarch64", - fullVersion: "9.6.20260210-0", - token: "9.6", - ok: true, - }, - { - name: "ROSA format", - description: "rhcos-9.6.20250701-0-x86_64", - fullVersion: "9.6.20250701-0", - token: "9.6", - ok: true, - }, - { - name: "pre-RHEL-aligned old OCP format", - description: "OpenShift 4.18 418.94.202511191518-0 x86_64", - fullVersion: "418.94.202511191518-0", - token: "418.94", - ok: true, - }, - { - name: "no version in description", - description: "some random description without a version", - ok: false, - }, - { - name: "empty description", - description: "", - ok: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - fullVersion, token, ok := extractVersionFromDescription(tc.description) - assert.Equal(t, tc.ok, ok) - assert.Equal(t, tc.fullVersion, fullVersion) - assert.Equal(t, tc.token, token) - }) - } -} - -func TestCmpVersionToken(t *testing.T) { - cases := []struct { - name string - a, b string - sign int // negative, zero, or positive - }{ - {"equal tokens", "9.6", "9.6", 0}, - {"a less than b (minor)", "9.5", "9.6", -1}, - {"a greater than b (minor)", "9.7", "9.6", 1}, - {"a less than b (major)", "9.6", "10.0", -1}, - {"a greater than b (major)", "10.0", "9.6", 1}, - {"pre-RHEL-aligned vs RHEL-aligned", "418.94", "9.6", 1}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := cmpVersionToken(tc.a, tc.b) - switch { - case tc.sign < 0: - assert.Less(t, got, 0) - case tc.sign > 0: - assert.Greater(t, got, 0) - default: - assert.Equal(t, 0, got) - } - }) - } -} - -func TestCmpRHCOSVersion(t *testing.T) { - cases := []struct { - name string - a, b string - sign int - }{ - {"same major.minor, different date", "9.6.20260210-0", "9.6.20260110-0", 0}, - {"higher minor version", "9.6.20260210-0", "9.5.20251001-0", 1}, - {"lower minor version", "9.5.20251001-0", "9.6.20260210-0", -1}, - {"higher major version", "10.0.20260210-0", "9.6.20260210-0", 1}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := cmpRHCOSVersion(tc.a, tc.b) - switch { - case tc.sign < 0: - assert.Less(t, got, 0) - case tc.sign > 0: - assert.Greater(t, got, 0) - default: - assert.Equal(t, 0, got) - } - }) - } -} - -func TestIsPreRHELAlignedToken(t *testing.T) { - cases := []struct { - token string - expected bool - }{ - {"418.94", true}, - {"400.0", true}, - {"101.0", true}, - {"100.0", false}, - {"9.6", false}, - {"10.0", false}, - {"99.9", false}, - } - - for _, tc := range cases { - t.Run(tc.token, func(t *testing.T) { - assert.Equal(t, tc.expected, isPreRHELAlignedToken(tc.token)) - }) - } -} - func TestDetectAMIKind(t *testing.T) { const ( ocpProductID = "59ead7de-2540-4653-a8b0-fa7926d5c845" diff --git a/pkg/controller/bootimage/marketplace/marketplace.go b/pkg/controller/bootimage/marketplace/marketplace.go new file mode 100644 index 0000000000..7d8ac68257 --- /dev/null +++ b/pkg/controller/bootimage/marketplace/marketplace.go @@ -0,0 +1,121 @@ +// Package marketplace holds the AWS Marketplace RHCOS product catalog and the version-parsing/ +// comparison logic used to identify and validate Marketplace AMIs. It is dependency-light +// (stdlib only) so it can be imported both by the boot image controller (pkg/controller/bootimage) +// and by standalone tooling (e.g. devex/cmd/check-aws-marketplace-skew) without pulling in +// client-go/controller-runtime. +package marketplace + +import ( + "regexp" + "strconv" + "strings" +) + +// ROSAProductID is the marketplace product ID for ROSA Classic. +const ROSAProductID = "34850061-abaf-402d-92df-94325c9e947f" + +// Products maps AWS Marketplace product IDs to human-readable variant names. +// These IDs are stable — they are tied to marketplace listings and will not change. +var Products = map[string]string{ + // x86_64 + "59ead7de-2540-4653-a8b0-fa7926d5c845": "OCP x86_64", + "963b36c3-de6f-48ed-b802-2b38b2a2cdeb": "OKE x86_64", + "f5da01a6-d046-487c-9072-42fe53b1cad4": "OPP x86_64", + // arm64 + "abc249f8-7440-45f7-a4b1-c026baff64c1": "OCP arm64", + "d2d3ebcd-c1ca-43d8-bf0a-530433200f35": "OKE arm64", + "be6d3e94-c8dc-4a3e-9218-4b449b11f06f": "OPP arm64", + // x86_64 EMEA + "962791c7-3ae5-46d1-ba62-c7a5ebac54fd": "OCP EMEA x86_64", + "7026c8d7-392c-4010-b93c-f93f7bc5495f": "OKE EMEA x86_64", + "628c9df3-0254-4f91-bc1f-8619d1b8eaa8": "OPP EMEA x86_64", + // ROSA + ROSAProductID: "ROSA", +} + +// ProductName returns the human-readable variant name for a marketplace product ID, +// falling back to the product ID itself if it is not in the map. +func ProductName(productID string) string { + if name, ok := Products[productID]; ok { + return name + } + return productID +} + +var productIDRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// ExtractProductID returns the trailing UUID-format product ID from a marketplace AMI name, e.g.: +// +// RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-{product-id} +// +// Returns an empty string if no valid product ID is found. +func ExtractProductID(name string) string { + parts := strings.Split(name, "-") + if len(parts) < 5 { + return "" + } + candidate := strings.Join(parts[len(parts)-5:], "-") + if productIDRegex.MatchString(candidate) { + return candidate + } + return "" +} + +// descriptionVersionRe matches the full RHCOS release string embedded in marketplace AMI descriptions. +// The full match is the API-valid release string; group 1 is the N.M token for version comparison. +// Both description formats in use embed the full release string inline: +// - RHEL marketplace: "RHEL CoreOS 9.6 9.6.20260210-0 x86_64" → "9.6.20260210-0" / "9.6" +// - ROSA: "rhcos-9.6.20250701-0-x86_64" → "9.6.20250701-0" / "9.6" +var descriptionVersionRe = regexp.MustCompile(`(\d+\.\d+)\.(?:[0-9]{8}|[0-9]{12})-\d+`) + +// ExtractVersionFromDescription parses the RHCOS release string from a marketplace AMI description. +// Returns the full release string (e.g. "9.6.20260210-0") suitable for ClusterBootImageAutomatic.RHCOSVersion, +// the N.M token (e.g. "9.6") for version comparison, and whether parsing succeeded. +func ExtractVersionFromDescription(description string) (fullVersion, token string, ok bool) { + m := descriptionVersionRe.FindStringSubmatch(description) + if m == nil { + return "", "", false + } + return m[0], m[1], true +} + +// CmpRHCOSVersion compares two full RHCOS release strings (e.g. "9.6.20260210-0") by their +// major.minor version only. Returns negative if a < b, zero if equal, positive if a > b. +func CmpRHCOSVersion(a, b string) int { + tokenOf := func(v string) string { + p := strings.SplitN(v, ".", 3) + if len(p) < 2 { + return v + } + return p[0] + "." + p[1] + } + return CmpVersionToken(tokenOf(a), tokenOf(b)) +} + +// CmpVersionToken compares two "major.minor" version tokens. +// Returns negative if a < b, zero if equal, positive if a > b. +func CmpVersionToken(a, b string) int { + parse := func(s string) (int, int) { + parts := strings.SplitN(s, ".", 2) + if len(parts) != 2 { + return 0, 0 + } + major, _ := strconv.Atoi(parts[0]) + minor, _ := strconv.Atoi(parts[1]) + return major, minor + } + aMaj, aMin := parse(a) + bMaj, bMin := parse(b) + if aMaj != bMaj { + return aMaj - bMaj + } + return aMin - bMin +} + +// IsPreRHELAlignedToken reports whether a version token uses the pre-4.19 OCP-based RHCOS +// versioning scheme (e.g. "418.94") rather than the RHEL-aligned scheme (e.g. "9.6"). +// Pre-RHEL-aligned tokens have a major component > 100 (encoding the OCP major version * 100 + minor). +func IsPreRHELAlignedToken(token string) bool { + major, _ := strconv.Atoi(strings.SplitN(token, ".", 2)[0]) + return major > 100 +} diff --git a/pkg/controller/bootimage/marketplace/marketplace_test.go b/pkg/controller/bootimage/marketplace/marketplace_test.go new file mode 100644 index 0000000000..520b59084e --- /dev/null +++ b/pkg/controller/bootimage/marketplace/marketplace_test.go @@ -0,0 +1,192 @@ +package marketplace + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtractProductID(t *testing.T) { + cases := []struct { + name string + amiName string + expected string + }{ + { + name: "OCP x86_64 AMI", + amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-59ead7de-2540-4653-a8b0-fa7926d5c845", + expected: "59ead7de-2540-4653-a8b0-fa7926d5c845", + }, + { + name: "OCP arm64 AMI", + amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-aarch64-0-abc249f8-7440-45f7-a4b1-c026baff64c1", + expected: "abc249f8-7440-45f7-a4b1-c026baff64c1", + }, + { + name: "ROSA AMI", + amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-34850061-abaf-402d-92df-94325c9e947f", + expected: "34850061-abaf-402d-92df-94325c9e947f", + }, + { + name: "standard RHCOS AMI has no product ID", + amiName: "rhcos-419-94-202504151514-0-x86_64", + expected: "", + }, + { + name: "name too short", + amiName: "short", + expected: "", + }, + { + name: "trailing segment not a UUID", + amiName: "RHEL-9.4-RHCOS-9.6_HVM_GA-20260210-x86_64-0-notauuid", + expected: "", + }, + { + name: "empty name", + amiName: "", + expected: "", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, ExtractProductID(tc.amiName)) + }) + } +} + +func TestExtractVersionFromDescription(t *testing.T) { + cases := []struct { + name string + description string + fullVersion string + token string + ok bool + }{ + { + name: "RHEL marketplace format x86_64", + description: "RHEL CoreOS 9.6 9.6.20260210-0 x86_64", + fullVersion: "9.6.20260210-0", + token: "9.6", + ok: true, + }, + { + name: "RHEL marketplace format aarch64", + description: "RHEL CoreOS 9.6 9.6.20260210-0 aarch64", + fullVersion: "9.6.20260210-0", + token: "9.6", + ok: true, + }, + { + name: "ROSA format", + description: "rhcos-9.6.20250701-0-x86_64", + fullVersion: "9.6.20250701-0", + token: "9.6", + ok: true, + }, + { + name: "pre-RHEL-aligned old OCP format", + description: "OpenShift 4.18 418.94.202511191518-0 x86_64", + fullVersion: "418.94.202511191518-0", + token: "418.94", + ok: true, + }, + { + name: "no version in description", + description: "some random description without a version", + ok: false, + }, + { + name: "empty description", + description: "", + ok: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fullVersion, token, ok := ExtractVersionFromDescription(tc.description) + assert.Equal(t, tc.ok, ok) + assert.Equal(t, tc.fullVersion, fullVersion) + assert.Equal(t, tc.token, token) + }) + } +} + +func TestCmpVersionToken(t *testing.T) { + cases := []struct { + name string + a, b string + sign int // negative, zero, or positive + }{ + {"equal tokens", "9.6", "9.6", 0}, + {"a less than b (minor)", "9.5", "9.6", -1}, + {"a greater than b (minor)", "9.7", "9.6", 1}, + {"a less than b (major)", "9.6", "10.0", -1}, + {"a greater than b (major)", "10.0", "9.6", 1}, + {"pre-RHEL-aligned vs RHEL-aligned", "418.94", "9.6", 1}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := CmpVersionToken(tc.a, tc.b) + switch { + case tc.sign < 0: + assert.Less(t, got, 0) + case tc.sign > 0: + assert.Greater(t, got, 0) + default: + assert.Equal(t, 0, got) + } + }) + } +} + +func TestCmpRHCOSVersion(t *testing.T) { + cases := []struct { + name string + a, b string + sign int + }{ + {"same major.minor, different date", "9.6.20260210-0", "9.6.20260110-0", 0}, + {"higher minor version", "9.6.20260210-0", "9.5.20251001-0", 1}, + {"lower minor version", "9.5.20251001-0", "9.6.20260210-0", -1}, + {"higher major version", "10.0.20260210-0", "9.6.20260210-0", 1}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := CmpRHCOSVersion(tc.a, tc.b) + switch { + case tc.sign < 0: + assert.Less(t, got, 0) + case tc.sign > 0: + assert.Greater(t, got, 0) + default: + assert.Equal(t, 0, got) + } + }) + } +} + +func TestIsPreRHELAlignedToken(t *testing.T) { + cases := []struct { + token string + expected bool + }{ + {"418.94", true}, + {"400.0", true}, + {"101.0", true}, + {"100.0", false}, + {"9.6", false}, + {"10.0", false}, + {"99.9", false}, + } + + for _, tc := range cases { + t.Run(tc.token, func(t *testing.T) { + assert.Equal(t, tc.expected, IsPreRHELAlignedToken(tc.token)) + }) + } +} diff --git a/pkg/controller/bootimage/ms_helpers.go b/pkg/controller/bootimage/ms_helpers.go index 630ea1867c..6925679853 100644 --- a/pkg/controller/bootimage/ms_helpers.go +++ b/pkg/controller/bootimage/ms_helpers.go @@ -23,6 +23,8 @@ import ( archtranslater "github.com/coreos/stream-metadata-go/arch" "github.com/coreos/stream-metadata-go/stream" corev1 "k8s.io/api/core/v1" + + "github.com/openshift/machine-config-operator/pkg/controller/bootimage/marketplace" ) // syncMAPIMachineSets will attempt to enqueue every machineset @@ -105,7 +107,7 @@ func (ctrl *Controller) syncMAPIMachineSets(reason string) { if version != "" { // Keep the oldest (lowest) RHCOS version across all marketplace MachineSets so that // skew enforcement uses the most conservative baseline rather than the last value. - if rhcosVersion == "" || cmpRHCOSVersion(version, rhcosVersion) < 0 { + if rhcosVersion == "" || marketplace.CmpRHCOSVersion(version, rhcosVersion) < 0 { rhcosVersion = version } } diff --git a/pkg/controller/bootimage/platform_helpers.go b/pkg/controller/bootimage/platform_helpers.go index 89e9904db8..92e2eba53d 100644 --- a/pkg/controller/bootimage/platform_helpers.go +++ b/pkg/controller/bootimage/platform_helpers.go @@ -15,6 +15,8 @@ import ( osconfigv1 "github.com/openshift/api/config/v1" machinev1beta1 "github.com/openshift/api/machine/v1beta1" + + "github.com/openshift/machine-config-operator/pkg/controller/bootimage/marketplace" ) // AzureVariant represents the different Azure marketplace image variants @@ -189,7 +191,7 @@ func reconcileAWSProviderSpec(streamData *stream.Stream, arch string, _ *osconfi newAMI = awsRegionImage.Image case amiKindMarketplace: - klog.Infof("MachineSet %s: detected marketplace AMI %s (%s)", machineSetName, currentAMI, productName(productID)) + klog.Infof("MachineSet %s: detected marketplace AMI %s (%s)", machineSetName, currentAMI, marketplace.ProductName(productID)) newAMI, rhcosVersion, err = resolveMarketplaceAMI(ctx, ec2Client, streamData, arch, productID, machineSetName) if err != nil { return false, false, nil, "", err @@ -199,7 +201,7 @@ func reconcileAWSProviderSpec(streamData *stream.Stream, arch string, _ *osconfi } case amiKindROSA: - klog.Infof("MachineSet %s: detected ROSA marketplace AMI %s (%s)", machineSetName, currentAMI, productName(productID)) + klog.Infof("MachineSet %s: detected ROSA marketplace AMI %s (%s)", machineSetName, currentAMI, marketplace.ProductName(productID)) newAMI, rhcosVersion, err = resolveMarketplaceAMI(ctx, ec2Client, streamData, arch, productID, machineSetName) if err != nil { return false, false, nil, "", err