feat(release): add force option to delete/recreate dangling tags and releases (#7119)

.github/scripts/bulk_release.py

@@ -138,6 +138,10 @@ def discover_all_connectors(repo_root: Path) -> list[Connector]:
                 continue
             if not (connector_path / "Dockerfile").exists():
                 continue
+            if not (
+                connector_path / "__metadata__" / "connector_manifest.json"
+            ).exists():
+                continue
             connectors.append(
                 Connector(
                     name=connector_path.name,

.github/workflows/release-bulk-connectors.yml

@@ -8,11 +8,12 @@ run-name: >-
   ${{
     github.event_name == 'push'
       && format('🚀 bulk (platform tag {0})', github.ref_name)
-      || format('🚀 bulk: {0}{1}{2}{3}',
+      || format('🚀 bulk: {0}{1}{2}{3}{4}',
            inputs.connectors,
            inputs.version != '' && format('@{0}', inputs.version) || '',
            inputs.dry_run && ' · dry-run' || '',
-           inputs.manifest_only && ' · manifest-only' || '')
+           inputs.manifest_only && ' · manifest-only' || '',
+           inputs.force && ' · force' || '')
   }}
 
 # Bulk release orchestrator.
@@ -85,6 +86,15 @@ on:
         required: false
         type: boolean
         default: true
+      force:
+        description: >
+          Force — for every dispatched connector, delete any existing tag
+          and/or GitHub Release for this version before recreating them from
+          scratch. Use this to recover from connectors left with a dangling
+          tag/release after a partially failed release.
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
   group: release-bulk-connectors
@@ -122,6 +132,7 @@ jobs:
           INPUT_VERSION: ${{ inputs.version }}
           INPUT_DRY_RUN: ${{ inputs.dry_run }}
           INPUT_MANIFEST_ONLY: ${{ inputs.manifest_only }}
+          INPUT_FORCE: ${{ inputs.force }}
         run: |
           set -euo pipefail
 
@@ -133,12 +144,14 @@ jobs:
             echo "version=${TAG_NAME}" >> "$GITHUB_OUTPUT"
             echo "dry_run=false" >> "$GITHUB_OUTPUT"
             echo "manifest_only=true" >> "$GITHUB_OUTPUT"
+            echo "force=false" >> "$GITHUB_OUTPUT"
             echo "🏷️ Platform release tag: ${TAG_NAME} → bulk manifest-only run for all connectors"
           else
             echo "connectors=${INPUT_CONNECTORS}" >> "$GITHUB_OUTPUT"
             echo "version=${INPUT_VERSION}" >> "$GITHUB_OUTPUT"
             echo "dry_run=${INPUT_DRY_RUN}" >> "$GITHUB_OUTPUT"
             echo "manifest_only=${INPUT_MANIFEST_ONLY}" >> "$GITHUB_OUTPUT"
+            echo "force=${INPUT_FORCE}" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Resolve connectors and version
@@ -148,6 +161,7 @@ jobs:
           VERSION: ${{ steps.trigger.outputs.version }}
           DRY_RUN: ${{ steps.trigger.outputs.dry_run }}
           MANIFEST_ONLY: ${{ steps.trigger.outputs.manifest_only }}
+          FORCE: ${{ steps.trigger.outputs.force }}
         run: |
           set -euo pipefail
 
@@ -165,8 +179,11 @@ jobs:
           # A dry run previews the full intended set, including connectors already
           # released at this version. A real run (manifest-only or not) skips them,
           # since it creates a real tag/GitHub Release per connector and re-running
-          # for an already-released version would collide with the existing tag.
-          if [ "${DRY_RUN}" = "true" ]; then
+          # for an already-released version would collide with the existing tag —
+          # unless force is set, in which case those connectors must still be
+          # dispatched so release-connector.yml's own force logic can delete and
+          # recreate their tag/release.
+          if [ "${DRY_RUN}" = "true" ] || [ "${FORCE}" = "true" ]; then
             ARGS+=("--include-released")
           fi
 
@@ -188,6 +205,7 @@ jobs:
           CONNECTORS_JSON: ${{ steps.resolve.outputs.connectors }}
           VERSION: ${{ steps.resolve.outputs.version }}
           MANIFEST_ONLY: ${{ steps.trigger.outputs.manifest_only }}
+          FORCE: ${{ steps.trigger.outputs.force }}
           REF: ${{ github.ref_name }}
           REPO: ${{ github.repository }}
           DRY_RUN: ${{ steps.trigger.outputs.dry_run }}
@@ -207,6 +225,10 @@ jobs:
               echo "🧩 **Manifest-only mode** — Docker build/push is skipped for every connector (still handled by CircleCI during the transition). Tag, GitHub Release, manifest fragment, and container_version commit-back are still fully produced for each connector."
               echo ""
             fi
+            if [ "$FORCE" = "true" ]; then
+              echo "⚠️ **Force mode** — any existing tag/GitHub Release at this version will be deleted and recreated for every connector."
+              echo ""
+            fi
             echo "Dispatching connector release(s) on ref \`$REF\`."
             echo ""
             echo "| Connector | Target | Status | Track |"
@@ -228,7 +250,8 @@ jobs:
                  -f connector_name="$name" \
                  -f version="$VERSION" \
                  -f dry_run="$DRY_RUN" \
-                 -f manifest_only="$MANIFEST_ONLY"; then
+                 -f manifest_only="$MANIFEST_ONLY" \
+                 -f force="$FORCE"; then
               echo "| $name | $target_label | ✅ dispatched | [view run]($track_url) |" >> "$GITHUB_STEP_SUMMARY"
               ok=$((ok + 1))
             else
@@ -265,6 +288,7 @@ jobs:
           CONNECTORS_JSON: ${{ steps.resolve.outputs.connectors }}
           VERSION: ${{ steps.resolve.outputs.version }}
           MANIFEST_ONLY: ${{ steps.trigger.outputs.manifest_only }}
+          FORCE: ${{ steps.trigger.outputs.force }}
           DRY_RUN: ${{ steps.trigger.outputs.dry_run }}
           START_TIME: ${{ steps.dispatch.outputs.start_time }}
         run: |
@@ -288,12 +312,13 @@ jobs:
           mapfile -t NAMES < <(printf '%s' "$CONNECTORS_JSON" | jq -r '.[]')
 
           # Must match release-connector.yml's run-name template exactly:
-          # "🚀 {connector_name}@{version}[ · dry-run][ · manifest-only]"
+          # "🚀 {connector_name}@{version}[ · dry-run][ · manifest-only][ · force]"
           declare -A TITLE_OF
           for name in "${NAMES[@]}"; do
             title="🚀 ${name}@${VERSION}"
             [ "$DRY_RUN" = "true" ] && title="${title} · dry-run"
             [ "$MANIFEST_ONLY" = "true" ] && title="${title} · manifest-only"
+            [ "$FORCE" = "true" ] && title="${title} · force"
             TITLE_OF["$name"]="$title"
           done
 

.github/workflows/release-connector.yml

@@ -10,11 +10,12 @@ run-name: >-
   ${{
     github.event_name == 'push'
       && format('🚀 {0}', github.ref_name)
-      || format('🚀 {0}{1}{2}{3}',
+      || format('🚀 {0}{1}{2}{3}{4}',
            inputs.connector_name,
            inputs.version != '' && format('@{0}', inputs.version) || '',
            inputs.dry_run && ' · dry-run' || '',
-           inputs.manifest_only && ' · manifest-only' || '')
+           inputs.manifest_only && ' · manifest-only' || '',
+           inputs.force && ' · force' || '')
   }}
 
 # Production release pipeline for individual connectors.
@@ -75,6 +76,17 @@ on:
         required: false
         type: boolean
         default: false
+      force:
+        description: >
+          Force — delete any existing GitHub Release for this version before
+          recreating it. If a tag already exists for this version, it is
+          reused as-is rather than being deleted/recreated. Use this to
+          recover from a partially failed release (e.g. manifest fragment
+          generation or XTM Hub publish failure) that left a dangling
+          tag/release behind.
+        required: false
+        type: boolean
+        default: false
       sdk_ref:
         description: >
           Optional git ref to pin connectors-sdk to in the built image.
@@ -112,6 +124,7 @@ jobs:
       is_tag_push: ${{ steps.parse.outputs.is_tag_push }}
       dry_run: ${{ steps.parse.outputs.dry_run }}
       manifest_only: ${{ steps.parse.outputs.manifest_only }}
+      force: ${{ steps.parse.outputs.force }}
       tag: ${{ steps.version.outputs.tag }}
       matrix_json: ${{ steps.matrix.outputs.matrix_json }}
     steps:
@@ -133,13 +146,15 @@ jobs:
             echo "is_tag_push=true" >> "$GITHUB_OUTPUT"
             echo "dry_run=false" >> "$GITHUB_OUTPUT"
             echo "manifest_only=false" >> "$GITHUB_OUTPUT"
+            echo "force=false" >> "$GITHUB_OUTPUT"
             echo "🏷️ Tag push: $TAG → connector=$CONNECTOR_NAME version=$VERSION"
           else
             # workflow_dispatch: use inputs directly
             echo "connector_name=${{ inputs.connector_name }}" >> "$GITHUB_OUTPUT"
             echo "is_tag_push=false" >> "$GITHUB_OUTPUT"
             echo "dry_run=${{ inputs.dry_run }}" >> "$GITHUB_OUTPUT"
             echo "manifest_only=${{ inputs.manifest_only }}" >> "$GITHUB_OUTPUT"
+            echo "force=${{ inputs.force }}" >> "$GITHUB_OUTPUT"
             echo "📋 Manual dispatch: connector=${{ inputs.connector_name }}"
           fi
 
@@ -469,29 +484,45 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Overwrite guard
+        id: overwrite-guard
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FORCE: ${{ needs.resolve.outputs.force }}
         run: |
           TAG="${{ needs.resolve.outputs.tag }}"
           if gh release view "$TAG" --json tagName >/dev/null 2>&1; then
-            echo "::error::A GitHub Release already exists for tag '${TAG}'. Aborting to prevent overwrite."
-            exit 1
+            if [ "$FORCE" = "true" ]; then
+              echo "⚠️ Force enabled — deleting existing release for '${TAG}'"
+              gh release delete "$TAG" --yes
+            else
+              echo "::error::A GitHub Release already exists for tag '${TAG}'. Aborting to prevent overwrite. Re-run with 'force' enabled to delete and recreate it."
+              exit 1
+            fi
           fi
 
       - name: Create tag (manual dispatch only)
         if: needs.resolve.outputs.is_tag_push != 'true'
+        env:
+          FORCE: ${{ needs.resolve.outputs.force }}
         run: |
           TAG="${{ needs.resolve.outputs.tag }}"
-          echo "🏷️  Creating tag: $TAG"
 
-          if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then
-            echo "::error::Tag '$TAG' already exists. Use a different version or push the existing tag to trigger a release."
-            exit 1
+          # `git rev-parse refs/tags/...` can't be used here: this job's checkout
+          # doesn't fetch tags, so a remote-only tag would look absent locally.
+          # `ls-remote` checks the remote directly regardless of local state.
+          if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null 2>&1; then
+            if [ "$FORCE" = "true" ]; then
+              echo "⚠️ Force enabled — tag '${TAG}' already exists, reusing it as-is"
+            else
+              echo "::error::Tag '$TAG' already exists. Use a different version, push the existing tag to trigger a release, or re-run with 'force' enabled to reuse it."
+              exit 1
+            fi
+          else
+            echo "🏷️  Creating tag: $TAG"
+            git tag "$TAG"
+            git push origin "$TAG"
           fi
 
-          git tag "$TAG"
-          git push origin "$TAG"
-
       - name: Patch manifest container_version
         id: patch-manifest
         run: |
@@ -502,7 +533,7 @@ jobs:
           if [ ! -f "$MANIFEST" ]; then
             echo "patched=false" >> "$GITHUB_OUTPUT"
             echo "⚠️ Manifest not found: $MANIFEST — skipping version patch"
-            exit 0
+            exit 1
           fi
 
           jq --arg v "$VERSION" '.container_version = $v' "$MANIFEST" > "$RUNNER_TEMP/manifest.json" \
@@ -528,7 +559,7 @@ jobs:
 
           if [ ! -f "$MANIFEST" ]; then
             echo "::warning::Manifest not found for ${CONNECTOR_DIR} — skipping fragment generation"
-            exit 0
+            exit 1
           fi
 
           pip install --quiet -r shared/tools/composer/generate_manifest_fragment/requirements.txt
@@ -555,7 +586,7 @@ jobs:
 
           if [ -z "$XTM_HUB_TOKEN" ]; then
             echo "::warning::XTM_HUB_TOKEN secret not set -- skipping XTM Hub publish"
-            exit 0
+            exit 1
           fi
 
           QUERY='mutation IngestManifestFragments($manifestFragments: [ManifestFragmentInput!]!) {
@@ -587,12 +618,12 @@ jobs:
 
           if [ "$HTTP_STATUS" -ge 400 ]; then
             echo "::warning::XTM Hub publish failed with HTTP $HTTP_STATUS"
-            exit 0
+            exit 1
           fi
 
           if echo "$RESPONSE" | jq -e '.errors' >/dev/null 2>&1; then
             echo "::warning::XTM Hub returned errors: $(echo "$RESPONSE" | jq -c '.errors')"
-            exit 0
+            exit 1
           fi
 
           SUCCESS=$(echo "$RESPONSE" | jq -r '.data.ingestManifestFragments.success // false')
@@ -701,6 +732,23 @@ jobs:
 
           echo "✅ Release created: $TITLE"
 
+      # Runs on any failure in this job that happens after the overwrite guard
+      # passed (guard failing itself means a real prior release exists and
+      # must be left untouched). Without this, a mid-job failure (e.g.
+      # manifest fragment generation or XTM Hub publish) leaves a dangling
+      # tag/release that blocks every future retry, since "Create tag" and
+      # "Overwrite guard" would then refuse to proceed on the next run.
+      - name: Cleanup tag on failure
+        if: failure() && steps.overwrite-guard.conclusion == 'success'
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TAG="${{ needs.resolve.outputs.tag }}"
+          echo "::warning::Release job failed — deleting tag '${TAG}' and any partial release so a retry can recreate it cleanly."
+          gh release delete "$TAG" --yes 2>/dev/null || true
+          git push origin --delete "$TAG" 2>/dev/null || true
+
   # ──────────────────────────────────────────────────────────
   # 5. Commit manifest version back to default branch
   # ──────────────────────────────────────────────────────────
@@ -730,7 +778,7 @@ jobs:
 
           if [ ! -f "$MANIFEST" ]; then
             echo "⚠️ Manifest not found: $MANIFEST — skipping"
-            exit 0
+            exit 1
           fi
 
           # Patch container_version
@@ -779,6 +827,7 @@ jobs:
         run: |
           DRY_RUN="${{ needs.resolve.outputs.dry_run }}"
           MANIFEST_ONLY="${{ needs.resolve.outputs.manifest_only }}"
+          FORCE="${{ needs.resolve.outputs.force }}"
           CONNECTOR="${{ needs.resolve.outputs.connector_name }}"
           VERSION="${{ needs.resolve.outputs.version }}"
           DIR="${{ needs.resolve.outputs.connector_dir }}"
@@ -808,6 +857,7 @@ jobs:
             echo "| Trigger | $([ \"$IS_TAG_PUSH\" = \"true\" ] && echo \"tag push\" || echo \"manual dispatch\") |"
             echo "| Dry Run | ${DRY_RUN} |"
             echo "| Manifest Only | ${MANIFEST_ONLY} |"
+            echo "| Force | ${FORCE} |"
             echo ""
 
             echo "### Job Results"