diff --git a/.github/scripts/bulk_release.py b/.github/scripts/bulk_release.py index 9a67c4757a9..af7da0788bd 100644 --- a/.github/scripts/bulk_release.py +++ b/.github/scripts/bulk_release.py @@ -138,6 +138,10 @@ def discover_all_connectors(repo_root: Path) -> list[Connector]: continue if not (connector_path / "Dockerfile").exists(): continue + if not ( + connector_path / "__metadata__" / "connector_manifest.json" + ).exists(): + continue connectors.append( Connector( name=connector_path.name, diff --git a/.github/workflows/release-bulk-connectors.yml b/.github/workflows/release-bulk-connectors.yml index d595b697249..0cdbd9347f6 100644 --- a/.github/workflows/release-bulk-connectors.yml +++ b/.github/workflows/release-bulk-connectors.yml @@ -8,11 +8,12 @@ run-name: >- ${{ github.event_name == 'push' && format('๐Ÿš€ bulk (platform tag {0})', github.ref_name) - || format('๐Ÿš€ bulk: {0}{1}{2}{3}', + || format('๐Ÿš€ bulk: {0}{1}{2}{3}{4}', inputs.connectors, inputs.version != '' && format('@{0}', inputs.version) || '', inputs.dry_run && ' ยท dry-run' || '', - inputs.manifest_only && ' ยท manifest-only' || '') + inputs.manifest_only && ' ยท manifest-only' || '', + inputs.force && ' ยท force' || '') }} # Bulk release orchestrator. @@ -85,6 +86,15 @@ on: required: false type: boolean default: true + force: + description: > + Force โ€” for every dispatched connector, delete any existing tag + and/or GitHub Release for this version before recreating them from + scratch. Use this to recover from connectors left with a dangling + tag/release after a partially failed release. + required: false + type: boolean + default: false concurrency: group: release-bulk-connectors @@ -122,6 +132,7 @@ jobs: INPUT_VERSION: ${{ inputs.version }} INPUT_DRY_RUN: ${{ inputs.dry_run }} INPUT_MANIFEST_ONLY: ${{ inputs.manifest_only }} + INPUT_FORCE: ${{ inputs.force }} run: | set -euo pipefail @@ -133,12 +144,14 @@ jobs: echo "version=${TAG_NAME}" >> "$GITHUB_OUTPUT" echo "dry_run=false" >> "$GITHUB_OUTPUT" echo "manifest_only=true" >> "$GITHUB_OUTPUT" + echo "force=false" >> "$GITHUB_OUTPUT" echo "๐Ÿท๏ธ Platform release tag: ${TAG_NAME} โ†’ bulk manifest-only run for all connectors" else echo "connectors=${INPUT_CONNECTORS}" >> "$GITHUB_OUTPUT" echo "version=${INPUT_VERSION}" >> "$GITHUB_OUTPUT" echo "dry_run=${INPUT_DRY_RUN}" >> "$GITHUB_OUTPUT" echo "manifest_only=${INPUT_MANIFEST_ONLY}" >> "$GITHUB_OUTPUT" + echo "force=${INPUT_FORCE}" >> "$GITHUB_OUTPUT" fi - name: Resolve connectors and version @@ -148,6 +161,7 @@ jobs: VERSION: ${{ steps.trigger.outputs.version }} DRY_RUN: ${{ steps.trigger.outputs.dry_run }} MANIFEST_ONLY: ${{ steps.trigger.outputs.manifest_only }} + FORCE: ${{ steps.trigger.outputs.force }} run: | set -euo pipefail @@ -165,8 +179,11 @@ jobs: # A dry run previews the full intended set, including connectors already # released at this version. A real run (manifest-only or not) skips them, # since it creates a real tag/GitHub Release per connector and re-running - # for an already-released version would collide with the existing tag. - if [ "${DRY_RUN}" = "true" ]; then + # for an already-released version would collide with the existing tag โ€” + # unless force is set, in which case those connectors must still be + # dispatched so release-connector.yml's own force logic can delete and + # recreate their tag/release. + if [ "${DRY_RUN}" = "true" ] || [ "${FORCE}" = "true" ]; then ARGS+=("--include-released") fi @@ -188,6 +205,7 @@ jobs: CONNECTORS_JSON: ${{ steps.resolve.outputs.connectors }} VERSION: ${{ steps.resolve.outputs.version }} MANIFEST_ONLY: ${{ steps.trigger.outputs.manifest_only }} + FORCE: ${{ steps.trigger.outputs.force }} REF: ${{ github.ref_name }} REPO: ${{ github.repository }} DRY_RUN: ${{ steps.trigger.outputs.dry_run }} @@ -207,6 +225,10 @@ jobs: echo "๐Ÿงฉ **Manifest-only mode** โ€” Docker build/push is skipped for every connector (still handled by CircleCI during the transition). Tag, GitHub Release, manifest fragment, and container_version commit-back are still fully produced for each connector." echo "" fi + if [ "$FORCE" = "true" ]; then + echo "โš ๏ธ **Force mode** โ€” any existing tag/GitHub Release at this version will be deleted and recreated for every connector." + echo "" + fi echo "Dispatching connector release(s) on ref \`$REF\`." echo "" echo "| Connector | Target | Status | Track |" @@ -228,7 +250,8 @@ jobs: -f connector_name="$name" \ -f version="$VERSION" \ -f dry_run="$DRY_RUN" \ - -f manifest_only="$MANIFEST_ONLY"; then + -f manifest_only="$MANIFEST_ONLY" \ + -f force="$FORCE"; then echo "| $name | $target_label | โœ… dispatched | [view run]($track_url) |" >> "$GITHUB_STEP_SUMMARY" ok=$((ok + 1)) else @@ -265,6 +288,7 @@ jobs: CONNECTORS_JSON: ${{ steps.resolve.outputs.connectors }} VERSION: ${{ steps.resolve.outputs.version }} MANIFEST_ONLY: ${{ steps.trigger.outputs.manifest_only }} + FORCE: ${{ steps.trigger.outputs.force }} DRY_RUN: ${{ steps.trigger.outputs.dry_run }} START_TIME: ${{ steps.dispatch.outputs.start_time }} run: | @@ -288,12 +312,13 @@ jobs: mapfile -t NAMES < <(printf '%s' "$CONNECTORS_JSON" | jq -r '.[]') # Must match release-connector.yml's run-name template exactly: - # "๐Ÿš€ {connector_name}@{version}[ ยท dry-run][ ยท manifest-only]" + # "๐Ÿš€ {connector_name}@{version}[ ยท dry-run][ ยท manifest-only][ ยท force]" declare -A TITLE_OF for name in "${NAMES[@]}"; do title="๐Ÿš€ ${name}@${VERSION}" [ "$DRY_RUN" = "true" ] && title="${title} ยท dry-run" [ "$MANIFEST_ONLY" = "true" ] && title="${title} ยท manifest-only" + [ "$FORCE" = "true" ] && title="${title} ยท force" TITLE_OF["$name"]="$title" done diff --git a/.github/workflows/release-connector.yml b/.github/workflows/release-connector.yml index 5b7be385d59..b7f89327488 100644 --- a/.github/workflows/release-connector.yml +++ b/.github/workflows/release-connector.yml @@ -10,11 +10,12 @@ run-name: >- ${{ github.event_name == 'push' && format('๐Ÿš€ {0}', github.ref_name) - || format('๐Ÿš€ {0}{1}{2}{3}', + || format('๐Ÿš€ {0}{1}{2}{3}{4}', inputs.connector_name, inputs.version != '' && format('@{0}', inputs.version) || '', inputs.dry_run && ' ยท dry-run' || '', - inputs.manifest_only && ' ยท manifest-only' || '') + inputs.manifest_only && ' ยท manifest-only' || '', + inputs.force && ' ยท force' || '') }} # Production release pipeline for individual connectors. @@ -75,6 +76,17 @@ on: required: false type: boolean default: false + force: + description: > + Force โ€” delete any existing GitHub Release for this version before + recreating it. If a tag already exists for this version, it is + reused as-is rather than being deleted/recreated. Use this to + recover from a partially failed release (e.g. manifest fragment + generation or XTM Hub publish failure) that left a dangling + tag/release behind. + required: false + type: boolean + default: false sdk_ref: description: > Optional git ref to pin connectors-sdk to in the built image. @@ -112,6 +124,7 @@ jobs: is_tag_push: ${{ steps.parse.outputs.is_tag_push }} dry_run: ${{ steps.parse.outputs.dry_run }} manifest_only: ${{ steps.parse.outputs.manifest_only }} + force: ${{ steps.parse.outputs.force }} tag: ${{ steps.version.outputs.tag }} matrix_json: ${{ steps.matrix.outputs.matrix_json }} steps: @@ -133,6 +146,7 @@ jobs: echo "is_tag_push=true" >> "$GITHUB_OUTPUT" echo "dry_run=false" >> "$GITHUB_OUTPUT" echo "manifest_only=false" >> "$GITHUB_OUTPUT" + echo "force=false" >> "$GITHUB_OUTPUT" echo "๐Ÿท๏ธ Tag push: $TAG โ†’ connector=$CONNECTOR_NAME version=$VERSION" else # workflow_dispatch: use inputs directly @@ -140,6 +154,7 @@ jobs: echo "is_tag_push=false" >> "$GITHUB_OUTPUT" echo "dry_run=${{ inputs.dry_run }}" >> "$GITHUB_OUTPUT" echo "manifest_only=${{ inputs.manifest_only }}" >> "$GITHUB_OUTPUT" + echo "force=${{ inputs.force }}" >> "$GITHUB_OUTPUT" echo "๐Ÿ“‹ Manual dispatch: connector=${{ inputs.connector_name }}" fi @@ -469,29 +484,45 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Overwrite guard + id: overwrite-guard env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FORCE: ${{ needs.resolve.outputs.force }} run: | TAG="${{ needs.resolve.outputs.tag }}" if gh release view "$TAG" --json tagName >/dev/null 2>&1; then - echo "::error::A GitHub Release already exists for tag '${TAG}'. Aborting to prevent overwrite." - exit 1 + if [ "$FORCE" = "true" ]; then + echo "โš ๏ธ Force enabled โ€” deleting existing release for '${TAG}'" + gh release delete "$TAG" --yes + else + echo "::error::A GitHub Release already exists for tag '${TAG}'. Aborting to prevent overwrite. Re-run with 'force' enabled to delete and recreate it." + exit 1 + fi fi - name: Create tag (manual dispatch only) if: needs.resolve.outputs.is_tag_push != 'true' + env: + FORCE: ${{ needs.resolve.outputs.force }} run: | TAG="${{ needs.resolve.outputs.tag }}" - echo "๐Ÿท๏ธ Creating tag: $TAG" - if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then - echo "::error::Tag '$TAG' already exists. Use a different version or push the existing tag to trigger a release." - exit 1 + # `git rev-parse refs/tags/...` can't be used here: this job's checkout + # doesn't fetch tags, so a remote-only tag would look absent locally. + # `ls-remote` checks the remote directly regardless of local state. + if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null 2>&1; then + if [ "$FORCE" = "true" ]; then + echo "โš ๏ธ Force enabled โ€” tag '${TAG}' already exists, reusing it as-is" + else + echo "::error::Tag '$TAG' already exists. Use a different version, push the existing tag to trigger a release, or re-run with 'force' enabled to reuse it." + exit 1 + fi + else + echo "๐Ÿท๏ธ Creating tag: $TAG" + git tag "$TAG" + git push origin "$TAG" fi - git tag "$TAG" - git push origin "$TAG" - - name: Patch manifest container_version id: patch-manifest run: | @@ -502,7 +533,7 @@ jobs: if [ ! -f "$MANIFEST" ]; then echo "patched=false" >> "$GITHUB_OUTPUT" echo "โš ๏ธ Manifest not found: $MANIFEST โ€” skipping version patch" - exit 0 + exit 1 fi jq --arg v "$VERSION" '.container_version = $v' "$MANIFEST" > "$RUNNER_TEMP/manifest.json" \ @@ -528,7 +559,7 @@ jobs: if [ ! -f "$MANIFEST" ]; then echo "::warning::Manifest not found for ${CONNECTOR_DIR} โ€” skipping fragment generation" - exit 0 + exit 1 fi pip install --quiet -r shared/tools/composer/generate_manifest_fragment/requirements.txt @@ -555,7 +586,7 @@ jobs: if [ -z "$XTM_HUB_TOKEN" ]; then echo "::warning::XTM_HUB_TOKEN secret not set -- skipping XTM Hub publish" - exit 0 + exit 1 fi QUERY='mutation IngestManifestFragments($manifestFragments: [ManifestFragmentInput!]!) { @@ -587,12 +618,12 @@ jobs: if [ "$HTTP_STATUS" -ge 400 ]; then echo "::warning::XTM Hub publish failed with HTTP $HTTP_STATUS" - exit 0 + exit 1 fi if echo "$RESPONSE" | jq -e '.errors' >/dev/null 2>&1; then echo "::warning::XTM Hub returned errors: $(echo "$RESPONSE" | jq -c '.errors')" - exit 0 + exit 1 fi SUCCESS=$(echo "$RESPONSE" | jq -r '.data.ingestManifestFragments.success // false') @@ -701,6 +732,23 @@ jobs: echo "โœ… Release created: $TITLE" + # Runs on any failure in this job that happens after the overwrite guard + # passed (guard failing itself means a real prior release exists and + # must be left untouched). Without this, a mid-job failure (e.g. + # manifest fragment generation or XTM Hub publish) leaves a dangling + # tag/release that blocks every future retry, since "Create tag" and + # "Overwrite guard" would then refuse to proceed on the next run. + - name: Cleanup tag on failure + if: failure() && steps.overwrite-guard.conclusion == 'success' + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ needs.resolve.outputs.tag }}" + echo "::warning::Release job failed โ€” deleting tag '${TAG}' and any partial release so a retry can recreate it cleanly." + gh release delete "$TAG" --yes 2>/dev/null || true + git push origin --delete "$TAG" 2>/dev/null || true + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # 5. Commit manifest version back to default branch # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -730,7 +778,7 @@ jobs: if [ ! -f "$MANIFEST" ]; then echo "โš ๏ธ Manifest not found: $MANIFEST โ€” skipping" - exit 0 + exit 1 fi # Patch container_version @@ -779,6 +827,7 @@ jobs: run: | DRY_RUN="${{ needs.resolve.outputs.dry_run }}" MANIFEST_ONLY="${{ needs.resolve.outputs.manifest_only }}" + FORCE="${{ needs.resolve.outputs.force }}" CONNECTOR="${{ needs.resolve.outputs.connector_name }}" VERSION="${{ needs.resolve.outputs.version }}" DIR="${{ needs.resolve.outputs.connector_dir }}" @@ -808,6 +857,7 @@ jobs: echo "| Trigger | $([ \"$IS_TAG_PUSH\" = \"true\" ] && echo \"tag push\" || echo \"manual dispatch\") |" echo "| Dry Run | ${DRY_RUN} |" echo "| Manifest Only | ${MANIFEST_ONLY} |" + echo "| Force | ${FORCE} |" echo "" echo "### Job Results"