fix(release): verified, gated mirroring — the pipeline proves its own completeness (A4) (#201)

Root fixes for the 0.0.85 publish incident (one stuck gh upload → >1h hang
→ silently incomplete mirror → downstream install 404s):

- publish-ecosystem: timeout-minutes 20 (was unbounded, 6h ceiling).
- mirror_res.sh GH section gets the SAME protection its GitCode section
  always had: per-file timeout 300, post-upload download-200 verification,
  delete-and-reupload retry <=5 (the observed bad asset state cleared on
  re-upload). Both sections now skip assets already serving 200, so
  incident-recovery re-runs are seconds, not a full re-mirror.
- Completeness gate verifies every asset on every ENABLED host and its
  non-zero exit is now BLOCKING in release.yml — the old
  '|| echo non-blocking' swallowed exactly this signal on 0.0.85. The
  index bump step is blocking for the same reason.
- ci-fresh-install: new wait-index gate job — post-release runs poll
  xim-pkgindex's mcpp.lua for the released version (<=15min) instead of
  racing the bump-PR merge (0.0.85: failed 55s after merge).

Verified: bash -n + YAML parse; positive run against the complete 0.0.85
mirror (16/16 assets 200 on both hosts, gate exit 0, skip path exercised);
negative run with an injected missing asset exits 1 naming it.

Design: .agents/docs/2026-07-08-root-cause-remediation-design.md A4.

Author: Sunrisepeak

.github/tools/mirror_res.sh

@@ -49,27 +49,52 @@ for a in "${ASSETS[@]}"; do
   gh release download "v$VER" -R "$SRC_REPO" -D "$DL" -p "$a" 2>/dev/null || { echo "[mirror] FAIL: missing $a in $SRC_REPO v$VER" >&2; exit 1; }
 done
 
-# ── GitHub (gh --clobber, reliable) ───────────────────────────────
+# ── GitHub (per-file timeout + verify + delete-and-reupload retry) ──
+# A4 hardening: `gh release upload` was observed both HANGING (>1h on one
+# asset, 2026-07-08) and leaving a phantom asset that later 404s. Treat GH
+# exactly like GitCode below: bounded upload, verify the actual download,
+# retry with a delete first (the observed bad state cleared on re-upload).
+GH_ENABLED=0
 if [[ -n "${XLINGS_RES_TOKEN:-}" ]] || gh auth status >/dev/null 2>&1; then
+  GH_ENABLED=1
   info "GitHub $GH_DST tag $VER"
   GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release view "$VER" -R "$GH_DST" >/dev/null 2>&1 \
     || GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release create "$VER" -R "$GH_DST" --title "$VER" --notes "$PROJ $VER (mirror of $SRC_REPO)"
   for a in "${ASSETS[@]}"; do
-    GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber
+    # Idempotent re-runs (incident recovery) skip assets already serving 200.
+    if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://github.com/${GH_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
+      info "gh $a already mirrored (200), skipping"
+      continue
+    fi
+    for try in 1 2 3 4 5; do
+      GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout 300 gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber || true
+      if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://github.com/${GH_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
+        break
+      fi
+      echo "[mirror] gh $a not 200 after try $try; delete + reupload..."
+      GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release delete-asset "$VER" "$a" -R "$GH_DST" -y 2>/dev/null || true
+      sleep 4
+    done
   done
 else
   info "no github auth; skipping github mirror"
 fi
 
 # ── GitCode (gtc, per-file retry — multi-file upload can 502 and drop files) ──
+GTC_ENABLED=0
 if [[ -n "${GITCODE_TOKEN:-}" ]] && command -v gtc >/dev/null 2>&1; then
+  GTC_ENABLED=1
   info "GitCode $GTC_DST tag $VER"
   gtc release create "$GTC_DST" --tag "$VER" --name "$VER" 2>/dev/null || true
   # Upload then verify the actual DOWNLOAD is 200 (gtc can report success yet
   # leave a phantom/missing asset — obs_callback flakiness), retry up to 5.
   for a in "${ASSETS[@]}"; do
+    if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
+      info "gtc $a already mirrored (200), skipping"
+      continue
+    fi
     for try in 1 2 3 4 5; do
-      gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" >/dev/null 2>&1 || true
+      timeout 300 gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" >/dev/null 2>&1 || true
       if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
         break
       fi
@@ -80,15 +105,21 @@ else
   info "no GITCODE_TOKEN/gtc; skipping gitcode mirror"
 fi
 
-# ── Verify every platform on both hosts ───────────────────────────
+# ── Completeness gate: verify every asset on every ENABLED host ──
+# A4: this is the mirror's definition of done. The caller must treat a
+# non-zero exit as a hard failure (release.yml does since the 0.0.85
+# incident — the old `|| echo non-blocking` swallowed exactly this).
 info "verify:"
 rc=0
-for host in "github.com/$GH_DST" "gitcode.com/$GTC_DST"; do
+hosts=()
+[[ "${GH_ENABLED:-0}"  == 1 ]] && hosts+=("github.com/$GH_DST")
+[[ "${GTC_ENABLED:-0}" == 1 ]] && hosts+=("gitcode.com/$GTC_DST")
+for host in "${hosts[@]}"; do
   for a in "${ASSETS[@]}"; do
     code=$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://${host}/releases/download/${VER}/${a}" 2>/dev/null || echo ERR)
     echo "  $code  https://${host}/releases/download/${VER}/${a}"
-    [[ "$code" == 200 ]] || rc=1
+    [[ "$code" == 200 ]] || { rc=1; echo "[mirror] FAIL: missing/unverified: https://${host}/releases/download/${VER}/${a}" >&2; }
   done
 done
-[[ $rc == 0 ]] && info "all platforms mirrored OK" || { echo "[mirror] WARN: some assets not 200" >&2; }
+[[ $rc == 0 ]] && info "all assets mirrored + verified on ${#hosts[@]} host(s)"
 exit $rc

.github/workflows/ci-fresh-install.yml

@@ -35,7 +35,36 @@ jobs:
   # ──────────────────────────────────────────────────────────────────
   # Linux: gcc@16.1.0, musl-gcc@15.1.0, llvm@20.1.7
   # ──────────────────────────────────────────────────────────────────
+  # A4: post-release runs race the xim-pkgindex bump (a PR a maintainer
+  # merges asynchronously) — the 0.0.85 fresh-install failed with
+  # "version not found: available 0.0.84" 55s after the bump merged.
+  # Convert the race into a bounded wait: poll the index's mcpp.lua until
+  # it carries the released version (<=15 min), then let every job run.
+  # Non-workflow_run triggers (manual, cron, PR) skip the wait.
+  wait-index:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
+    timeout-minutes: 20
+    steps:
+      - name: Wait for xim-pkgindex to track the released mcpp
+        if: ${{ github.event_name == 'workflow_run' }}
+        run: |
+          VER=$(curl -fsSL "https://api.github.com/repos/mcpp-community/mcpp/releases/latest" | python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")
+          echo "released: $VER — waiting for index..."
+          for i in $(seq 1 30); do
+            if curl -fsSL "https://raw.githubusercontent.com/openxlings/xim-pkgindex/main/pkgs/m/mcpp.lua" | grep -q "\"$VER\""; then
+              echo "index tracks $VER (after $((i*30))s)"; exit 0
+            fi
+            sleep 30
+          done
+          echo "::error::index never tracked $VER within 15min — merge the bump PR (openxlings/xim-pkgindex) and re-run"
+          exit 1
+      - name: No wait needed (manual/cron trigger)
+        if: ${{ github.event_name != 'workflow_run' }}
+        run: echo "not a post-release run; skipping index wait"
+
   linux-fresh:
+    needs: [wait-index]
     name: Linux fresh install
     if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-24.04
@@ -142,6 +171,7 @@ jobs:
   # hosts. Runs the released mcpp inside distro containers.
   # ──────────────────────────────────────────────────────────────────
   linux-distro-matrix:
+    needs: [wait-index]
     name: Linux distro (${{ matrix.distro }})
     if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-24.04
@@ -228,6 +258,7 @@ jobs:
   # macOS: llvm@20.1.7
   # ──────────────────────────────────────────────────────────────────
   macos-fresh:
+    needs: [wait-index]
     name: macOS fresh install
     if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
     # macos-14: the support floor (mcpp ≥0.0.50 / xlings ≥0.4.50 ship
@@ -289,6 +320,7 @@ jobs:
   # Windows: llvm@20.1.7 + MSVC STL
   # ──────────────────────────────────────────────────────────────────
   windows-fresh:
+    needs: [wait-index]
     name: Windows fresh install
     if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
     runs-on: windows-latest

.github/workflows/release.yml

@@ -707,6 +707,9 @@ jobs:
   publish-ecosystem:
     needs: [build-release, build-linux-aarch64, build-macos, build-windows]
     runs-on: ubuntu-latest
+    # A4 hardening: a single stuck upload once held this job >1h (6h default
+    # ceiling). The mirror script now has per-file timeouts, so 20min is ample.
+    timeout-minutes: 20
     env:
       XLINGS_RES_TOKEN: ${{ secrets.XLINGS_RES_TOKEN }}
       GITCODE_TOKEN: ${{ secrets.GITCODE_TOKEN }}
@@ -725,8 +728,10 @@ jobs:
         run: |
           chmod +x .github/tools/gtc .github/tools/mirror_res.sh
           export PATH="$PWD/.github/tools:$PATH"
-          bash .github/tools/mirror_res.sh mcpp "${{ steps.version.outputs.version }}" \
-            || echo "binary mirror reported issues (non-blocking)"
+          # A4: BLOCKING. Both mirror hosts serve users (GLOBAL + CN install
+          # paths); an incomplete mirror must fail here, visibly, instead of
+          # surfacing as a 404 in the first user's install (or fresh-install CI).
+          bash .github/tools/mirror_res.sh mcpp "${{ steps.version.outputs.version }}"
 
       - name: Open index bump PR (xim-pkgindex)
         if: ${{ env.XIM_PKGINDEX_TOKEN != '' }}
@@ -735,8 +740,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           chmod +x .github/tools/bump_index.sh
-          bash .github/tools/bump_index.sh mcpp "${{ steps.version.outputs.version }}" \
-            || echo "index bump reported issues (non-blocking)"
+          # A4: BLOCKING — a missing bump PR means the index never learns the
+          # release exists and every post-release install of the new version 404s.
+          bash .github/tools/bump_index.sh mcpp "${{ steps.version.outputs.version }}"
 
       # Post-release verification (ci-fresh-install) is triggered via its
       # `workflow_run: [release]` hook — a platform-generated event that is