diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73f1c569..d0398057 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ name: Release -run-name: ${{ github.event_name == 'workflow_dispatch' && inputs.governance_preflight_nonce != '' && format('Release governance preflight {0}', inputs.governance_preflight_nonce) || github.event_name == 'workflow_dispatch' && inputs.recover_release_version != '' && format('Release recovery {0} at {1} {2}', inputs.recover_release_version, inputs.recover_release_commit, inputs.recover_release_nonce) || github.event_name == 'workflow_dispatch' && inputs.repair_npm_version != '' && format('Release npm repair {0}', inputs.repair_npm_version) || github.event_name == 'workflow_dispatch' && inputs.repair_gitee_version != '' && format('Release Gitee repair {0}', inputs.repair_gitee_version) || github.workflow }} +run-name: ${{ github.event_name == 'workflow_dispatch' && inputs.governance_preflight_nonce != '' && format('Release governance preflight {0}', inputs.governance_preflight_nonce) || github.event_name == 'workflow_dispatch' && inputs.recover_release_version != '' && format('Release recovery {0} at {1} {2}', inputs.recover_release_version, inputs.recover_release_commit, inputs.recover_release_nonce) || github.event_name == 'workflow_dispatch' && inputs.repair_npm_version != '' && format('Release npm repair {0}', inputs.repair_npm_version) || github.event_name == 'workflow_dispatch' && inputs.repair_gitee_version != '' && format('Release Gitee repair {0}', inputs.repair_gitee_version) || github.event_name == 'workflow_dispatch' && inputs.repair_oss_version != '' && format('Release OSS repair {0}', inputs.repair_oss_version) || github.workflow }} on: push: @@ -15,6 +15,10 @@ on: description: "Mirror an existing public immutable release's assets to Gitee" required: false type: string + repair_oss_version: + description: "Mirror an existing public immutable release's assets to OSS" + required: false + type: string governance_preflight_commit: description: "Verify release governance for this exact default-branch commit without publishing" required: false @@ -75,6 +79,7 @@ jobs: env: REPAIR_NPM_VERSION: ${{ inputs.repair_npm_version }} REPAIR_GITEE_VERSION: ${{ inputs.repair_gitee_version }} + REPAIR_OSS_VERSION: ${{ inputs.repair_oss_version }} GOVERNANCE_COMMIT: ${{ inputs.governance_preflight_commit }} GOVERNANCE_NONCE: ${{ inputs.governance_preflight_nonce }} RECOVER_VERSION: ${{ inputs.recover_release_version }} @@ -88,17 +93,19 @@ jobs: set -eu npm_repair=0 gitee_repair=0 + oss_repair=0 governance=0 recovery=0 test -z "$REPAIR_NPM_VERSION" || npm_repair=1 test -z "$REPAIR_GITEE_VERSION" || gitee_repair=1 + test -z "$REPAIR_OSS_VERSION" || oss_repair=1 if test -n "$GOVERNANCE_COMMIT" || test -n "$GOVERNANCE_NONCE"; then governance=1; fi if test -n "$RECOVER_VERSION" || test -n "$RECOVER_TAG_OBJECT" || \ test -n "$RECOVER_COMMIT" || test -n "$RECOVER_FAILED_RUN_ID" || \ test -n "$RECOVER_FAILED_RUN_ATTEMPT" || \ test -n "$RECOVER_NONCE" || \ test -n "$RECOVER_CONFIRMATION"; then recovery=1; fi - test $((npm_repair + gitee_repair + governance + recovery)) -eq 1 || { + test $((npm_repair + gitee_repair + oss_repair + governance + recovery)) -eq 1 || { echo "workflow_dispatch must select exactly one release mode" >&2 exit 1 } @@ -106,6 +113,8 @@ jobs: echo "mode=repair_npm" >> "$GITHUB_OUTPUT" elif test "$gitee_repair" -eq 1; then echo "mode=repair_gitee" >> "$GITHUB_OUTPUT" + elif test "$oss_repair" -eq 1; then + echo "mode=repair_oss" >> "$GITHUB_OUTPUT" elif test "$governance" -eq 1; then if test -z "$GOVERNANCE_COMMIT" || test -z "$GOVERNANCE_NONCE"; then echo "governance preflight requires both commit and nonce" >&2 @@ -1878,7 +1887,7 @@ jobs: - publish-channels - mirror-gitee-release - repair-npm - - repair-gitee + - repair-channel runs-on: ubuntu-latest timeout-minutes: 2 permissions: {} @@ -1898,7 +1907,7 @@ jobs: PUBLISH_CHANNELS_RESULT: ${{ needs.publish-channels.result }} MIRROR_GITEE_RESULT: ${{ needs.mirror-gitee-release.result }} REPAIR_NPM_RESULT: ${{ needs.repair-npm.result }} - REPAIR_GITEE_RESULT: ${{ needs.repair-gitee.result }} + REPAIR_CHANNEL_RESULT: ${{ needs.repair-channel.result }} run: | set -eu require_result() { @@ -1928,7 +1937,7 @@ jobs: require_result authorize-recovery "$AUTHORIZE_RECOVERY_RESULT" skipped require_result governance-preflight "$GOVERNANCE_PREFLIGHT_RESULT" skipped require_result repair-npm "$REPAIR_NPM_RESULT" skipped - require_result repair-gitee "$REPAIR_GITEE_RESULT" skipped + require_result repair-channel "$REPAIR_CHANNEL_RESULT" skipped require_publication ;; workflow_dispatch:recover_release) @@ -1936,7 +1945,7 @@ jobs: require_result authorize-recovery "$AUTHORIZE_RECOVERY_RESULT" success require_result governance-preflight "$GOVERNANCE_PREFLIGHT_RESULT" skipped require_result repair-npm "$REPAIR_NPM_RESULT" skipped - require_result repair-gitee "$REPAIR_GITEE_RESULT" skipped + require_result repair-channel "$REPAIR_CHANNEL_RESULT" skipped require_publication ;; workflow_dispatch:governance_preflight) @@ -1950,7 +1959,7 @@ jobs: require_result publish-channels "$PUBLISH_CHANNELS_RESULT" skipped require_result mirror-gitee-release "$MIRROR_GITEE_RESULT" skipped require_result repair-npm "$REPAIR_NPM_RESULT" skipped - require_result repair-gitee "$REPAIR_GITEE_RESULT" skipped + require_result repair-channel "$REPAIR_CHANNEL_RESULT" skipped ;; workflow_dispatch:repair_npm) require_result dispatch-contract "$DISPATCH_RESULT" success @@ -1963,11 +1972,24 @@ jobs: require_result publish-release "$PUBLISH_RELEASE_RESULT" skipped require_result publish-channels "$PUBLISH_CHANNELS_RESULT" skipped require_result mirror-gitee-release "$MIRROR_GITEE_RESULT" skipped - require_result repair-gitee "$REPAIR_GITEE_RESULT" skipped + require_result repair-channel "$REPAIR_CHANNEL_RESULT" skipped ;; workflow_dispatch:repair_gitee) require_result dispatch-contract "$DISPATCH_RESULT" success - require_result repair-gitee "$REPAIR_GITEE_RESULT" success + require_result repair-channel "$REPAIR_CHANNEL_RESULT" success + require_result authorize-recovery "$AUTHORIZE_RECOVERY_RESULT" skipped + require_result governance-preflight "$GOVERNANCE_PREFLIGHT_RESULT" skipped + require_result release-contract "$RELEASE_CONTRACT_RESULT" skipped + require_result release "$RELEASE_RESULT" skipped + require_result verify-darwin-signatures "$DARWIN_SIGNATURE_RESULT" skipped + require_result publish-release "$PUBLISH_RELEASE_RESULT" skipped + require_result publish-channels "$PUBLISH_CHANNELS_RESULT" skipped + require_result mirror-gitee-release "$MIRROR_GITEE_RESULT" skipped + require_result repair-npm "$REPAIR_NPM_RESULT" skipped + ;; + workflow_dispatch:repair_oss) + require_result dispatch-contract "$DISPATCH_RESULT" success + require_result repair-channel "$REPAIR_CHANNEL_RESULT" success require_result authorize-recovery "$AUTHORIZE_RECOVERY_RESULT" skipped require_result governance-preflight "$GOVERNANCE_PREFLIGHT_RESULT" skipped require_result release-contract "$RELEASE_CONTRACT_RESULT" skipped @@ -1983,16 +2005,16 @@ jobs: exit 1 ;; esac - # Mirror an existing public immutable release's assets to Gitee on demand. The - # push-triggered mirror-gitee-release job depends on the same run's finalized - # dist artifact, so a tag that was already published (including one delivered - # by a recovery dispatch) cannot reuse it. This dispatch path re-derives the - # asset set from the immutable GitHub Release itself and mirrors it. The - # release tag and asset checksums remain the sealed authority boundary. - repair-gitee: - name: Mirror an existing immutable release to Gitee + # Repair one downstream channel from an existing public immutable release on + # demand. The push-triggered channel jobs depend on the original run's + # finalized dist artifact, so a tag that was already published cannot reuse + # it. This dispatch path re-derives the asset set from the immutable GitHub + # Release itself. The release tag, delivery provenance, and asset checksums + # remain the sealed authority boundary. + repair-channel: + name: Repair an existing immutable release channel needs: dispatch-contract - if: ${{ !cancelled() && needs.dispatch-contract.result == 'success' && needs.dispatch-contract.outputs.mode == 'repair_gitee' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.repository == 'DingTalk-Real-AI/dingtalk-workspace-cli' }} + if: ${{ !cancelled() && needs.dispatch-contract.result == 'success' && (needs.dispatch-contract.outputs.mode == 'repair_gitee' || needs.dispatch-contract.outputs.mode == 'repair_oss') && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.repository == 'DingTalk-Real-AI/dingtalk-workspace-cli' }} runs-on: ubuntu-latest timeout-minutes: 120 permissions: @@ -2010,7 +2032,7 @@ jobs: - name: Validate repair version working-directory: tooling env: - VERSION: ${{ inputs.repair_gitee_version }} + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} run: | set -eu . ./scripts/release/release-lib.sh @@ -2023,7 +2045,7 @@ jobs: id: authority uses: actions/github-script@v7 env: - VERSION: ${{ inputs.repair_gitee_version }} + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} with: script: | const { owner, repo } = context.repo; @@ -2102,12 +2124,20 @@ jobs: core.setOutput("commit_sha", commitSha); core.setOutput("tag_object", ref.data.object.sha); + - name: Check out sealed release source + uses: actions/checkout@v4 + with: + ref: ${{ steps.authority.outputs.commit_sha }} + path: release-source + persist-credentials: false + - name: Fetch and verify sealed release tag working-directory: tooling env: - VERSION: ${{ inputs.repair_gitee_version }} + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} RELEASE_COMMIT: ${{ steps.authority.outputs.commit_sha }} RELEASE_TAG_OBJECT: ${{ steps.authority.outputs.tag_object }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eu git fetch --force --no-tags \ @@ -2119,17 +2149,23 @@ jobs: - name: Require successful Release workflow delivery working-directory: tooling env: - VERSION: ${{ inputs.repair_gitee_version }} + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} RELEASE_COMMIT: ${{ steps.authority.outputs.commit_sha }} + REPAIR_MODE: ${{ needs.dispatch-contract.outputs.mode }} DWS_RELEASE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + case "$REPAIR_MODE" in + repair_gitee) target=gitee ;; + repair_oss) target=oss ;; + *) echo "Unsupported channel repair mode: $REPAIR_MODE" >&2; exit 2 ;; + esac ./scripts/release/verify-release-workflow-delivery.sh \ - "$VERSION" "$RELEASE_COMMIT" + --channel-repair "$target" "$VERSION" "$RELEASE_COMMIT" - name: Download and verify immutable GitHub Release assets env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ inputs.repair_gitee_version }} + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} run: | set -eu mkdir -p dist @@ -2143,14 +2179,31 @@ jobs: "$GITHUB_WORKSPACE/tooling/scripts/release/verify-release-artifacts.sh" "$VERSION" - name: Mirror release to Gitee (China) + if: ${{ needs.dispatch-contract.outputs.mode == 'repair_gitee' }} timeout-minutes: 100 working-directory: tooling run: | "$GITHUB_WORKSPACE/tooling/scripts/release/sync-to-gitee.sh" env: - VERSION: ${{ inputs.repair_gitee_version }} + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} DIST_DIR: ${{ github.workspace }}/dist GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }} GITEE_USER: ${{ secrets.GITEE_USER }} GITEE_REPO: ${{ secrets.GITEE_REPO }} DWS_REQUIRE_GITEE: "1" + + - name: Mirror release to OSS (China) + if: ${{ needs.dispatch-contract.outputs.mode == 'repair_oss' }} + timeout-minutes: 30 + working-directory: release-source + run: | + "$GITHUB_WORKSPACE/tooling/scripts/release/sync-to-oss.sh" + env: + VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} + DIST_DIR: ${{ github.workspace }}/dist + OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} + OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET }} + OSS_PREFIX: ${{ secrets.OSS_PREFIX }} + DWS_REQUIRE_OSS: "1" diff --git a/docs/automation.md b/docs/automation.md index 8c8a3742..fe107c56 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -108,6 +108,22 @@ commit, and failed tag-push run all match; it then reuses the normal release jobs. Do not put publication secrets in temporary branches or create ad-hoc recovery workflows. +If the immutable GitHub Release and npm package were delivered but a downstream +China mirror failed, dispatch the normal `Release` workflow from the protected +default branch with exactly one of `repair_gitee_version` or +`repair_oss_version`. Channel repair accepts a failed exact-tag run only when +its latest attempt completed the release contract, build, Apple signature, +immutable GitHub publication, and npm delivery checks for the exact tagged +commit. It then downloads and re-verifies the immutable assets before invoking +only the selected mirror. An OSS repair requires the OSS step itself to be the +recorded failure. A Gitee repair accepts either a failed Gitee job or a Gitee +job that was skipped behind that OSS failure; the latter is an explicit Gitee +backfill and does not claim that OSS has been repaired. Gitee repair requires +`GITEE_TOKEN`, `GITEE_USER`, and `GITEE_REPO`; OSS repair requires +`OSS_ACCESS_KEY_ID`, `OSS_ACCESS_KEY_SECRET`, `OSS_ENDPOINT`, and `OSS_BUCKET` +(with optional `OSS_PREFIX`) as Actions secrets. Missing credentials fail the +selected repair closed. + ## Handoff Checklist Before handoff, include: diff --git a/docs/releasing.md b/docs/releasing.md index fdee0cac..2c9ca532 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -97,7 +97,16 @@ dws-release v1.2.3 --from-beta v1.2.3-beta.1 --publish npm 补发只允许从默认分支触发 Release workflow 的 `repair_npm_version`。它只支持启用 immutable releases 后、由本流水线成功产出的公开 immutable release:目标必须是 `main` 历史中的 annotated tag,并且同 commit 的 `Build immutable GitHub Release` job 已成功。即使后续 npm 分发失败,这个独立的产物封存边界仍可作为补发依据。补发会用目标 commit 的 npm 模板重组包,逐平台核验资产和二进制版本,再发布到隔离的 `backfill` dist-tag,不会回滚 `latest` / `beta`。历史 mutable release 不进入自动补发路径,避免把可被替换的资产带入 npm。 -OSS/Gitee 分发失败时直接重跑该 tag 的 `Publish npm and mirrors` failed job;各步会复用 immutable GitHub 资产并保持 channel 单调。独立 Gitee release workflow 和本地直发脚本已停用,避免绕开 publication queue 或用重新构建的不同字节覆盖镜像。 +OSS/Gitee 分发失败且 GitHub immutable Release、npm 已交付时,从受保护的默认分支触发 +Release workflow,并且只填写 `repair_oss_version` 或 `repair_gitee_version` 之一。channel +repair 会精确绑定失败 tag run 的最新 attempt;contract、构建、Developer ID 签名、 +immutable GitHub 发布和 npm delivery 必须全部成功,且只能有一个 OSS/Gitee 下游失败, +随后才会下载并重新校验原始资产、修复所选镜像。OSS repair 必须匹配失败的 OSS step; +Gitee repair 还允许其 job 因该 OSS 失败而 skipped,此时只代表 Gitee backfill 成功, +不会把仍未修复的 OSS 标成成功。该证据不能用于 beta → stable 或 +stable baseline,后两者仍要求整条 Release 成功或受保护 recovery 成功。不要重跑旧 +attempt 的单个 failed job,以免在 attempts 之间拼接交付证据。独立 Gitee release +workflow 和本地直发脚本已停用,避免绕开 publication queue 或用重新构建的不同字节覆盖镜像。 ## 既有 tag 的紧急恢复 diff --git a/scripts/release/verify-release-workflow-delivery.sh b/scripts/release/verify-release-workflow-delivery.sh index f06397ee..40f19ca0 100755 --- a/scripts/release/verify-release-workflow-delivery.sh +++ b/scripts/release/verify-release-workflow-delivery.sh @@ -4,12 +4,28 @@ set -eu SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" . "$SCRIPT_DIR/release-lib.sh" +MODE="strict" +CHANNEL_REPAIR_TARGET="" +if [ "${1:-}" = "--channel-repair" ]; then + MODE="channel-repair" + shift + CHANNEL_REPAIR_TARGET="${1:-}" + shift + case "$CHANNEL_REPAIR_TARGET" in + oss|gitee) ;; + *) + printf 'channel repair target must be oss or gitee\n' >&2 + exit 2 + ;; + esac +fi + TAG="${1:-}" EXPECTED_COMMIT="${2:-}" REPOSITORY="${DWS_RELEASE_OFFICIAL_REPOSITORY:-DingTalk-Real-AI/dingtalk-workspace-cli}" [ -n "$TAG" ] && [ -n "$EXPECTED_COMMIT" ] || { - printf 'usage: verify-release-workflow-delivery.sh \n' >&2 + printf 'usage: verify-release-workflow-delivery.sh [--channel-repair ] \n' >&2 exit 2 } if ! release_is_stable_version "$TAG" && ! release_is_prerelease_version "$TAG"; then @@ -65,6 +81,200 @@ for run in runs: done } +find_failed_push_delivery() { + matches="" + page=1 + while :; do + page_result="$( + github_get "repos/$REPOSITORY/actions/workflows/release.yml/runs?branch=$TAG&event=push&status=completed&per_page=100&page=$page" \ + | python3 -c 'import json,sys +tag,commit,repository=sys.argv[1:] +runs=json.load(sys.stdin).get("workflow_runs", []) +print(len(runs)) +for run in runs: + if (run.get("head_sha") == commit and run.get("head_branch") == tag + and run.get("event") == "push" and run.get("status") == "completed" + and run.get("conclusion") == "failure" + and run.get("path") == ".github/workflows/release.yml" + and run.get("repository", {}).get("full_name") == repository): + run_id=run.get("id", "") + attempt=run.get("run_attempt", "") + if isinstance(run_id, int) and run_id > 0 and isinstance(attempt, int) and attempt > 0: + print(f"{run_id}\t{attempt}")' "$TAG" "$EXPECTED_COMMIT" "$REPOSITORY" + )" || return 1 + page_count="$(printf '%s\n' "$page_result" | sed -n '1p')" + page_matches="$(printf '%s\n' "$page_result" | sed '1d')" + matches="$(printf '%s\n%s\n' "$matches" "$page_matches" | sed '/^$/d')" + [ "$page_count" -eq 100 ] || break + page=$((page + 1)) + done + match_count="$(printf '%s\n' "$matches" | sed '/^$/d' | wc -l | tr -d ' ')" + [ "$match_count" -eq 1 ] || { + printf 'expected exactly one failed exact-tag push run for channel repair, found %s\n' \ + "$match_count" >&2 + return 1 + } + printf '%s\n' "$matches" +} + +verify_channel_repair_delivery() { + run_id="$1" + run_attempt="$2" + jobs_dir="$(mktemp -d "${TMPDIR:-/tmp}/dws-release-channel-jobs.XXXXXX")" + page=1 + while :; do + jobs_page="$jobs_dir/jobs-$page.json" + if ! github_get "repos/$REPOSITORY/actions/runs/$run_id/attempts/$run_attempt/jobs?per_page=100&page=$page" \ + >"$jobs_page"; then + rm -rf "$jobs_dir" + return 1 + fi + if ! page_count="$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1])).get("jobs", [])))' "$jobs_page")"; then + rm -rf "$jobs_dir" + return 1 + fi + [ "$page_count" -eq 100 ] || break + page=$((page + 1)) + done + + result=0 + python3 - "$EXPECTED_COMMIT" "$run_id" "$run_attempt" "$TAG" "$CHANNEL_REPAIR_TARGET" "$jobs_dir"/jobs-*.json <<'PY' || result=$? +import json +import sys + +commit, run_id, run_attempt, tag, target, *pages = sys.argv[1:] +jobs = [] +for page in pages: + with open(page, encoding="utf-8") as handle: + jobs.extend(json.load(handle).get("jobs", [])) + +def fail(message): + print( + f"failed Release run {run_id} attempt {run_attempt} is not safe " + f"channel-repair authority for {tag}: {message}", + file=sys.stderr, + ) + raise SystemExit(1) + +def one_job(name): + matches = [job for job in jobs if job.get("name") == name] + if len(matches) != 1: + fail(f"expected exactly one latest-attempt job {name!r}, found {len(matches)}") + job = matches[0] + if job.get("head_sha") != commit: + fail(f"job {name!r} is not bound to {commit}") + if job.get("status") != "completed": + fail(f"job {name!r} is not completed") + return job + +for name in ( + "release-contract", + "Build signed release artifacts", + "Verify Apple Developer ID signatures", + "Publish immutable GitHub Release", +): + if one_job(name).get("conclusion") != "success": + fail(f"required job {name!r} did not succeed") + +publish_release = one_job("Publish immutable GitHub Release") +immutable_steps = [ + step for step in publish_release.get("steps", []) + if step.get("name") == "Require immutable published GitHub Release" +] +if len(immutable_steps) != 1: + fail("expected exactly one immutable GitHub Release verification step") +if ( + immutable_steps[0].get("status") != "completed" + or immutable_steps[0].get("conclusion") != "success" +): + fail("immutable GitHub Release verification did not succeed") + +channels = one_job("Publish npm and mirrors") +if channels.get("conclusion") not in {"success", "failure"}: + fail("channel publication job was not completed with a conclusive result") + +steps = channels.get("steps", []) +for name in ( + "Download and verify immutable GitHub Release", + "Verify immutable npm package without publication credentials", + "Inspect npm channel state", + "Verify npm channel delivery", +): + matches = [step for step in steps if step.get("name") == name] + if len(matches) != 1: + fail(f"expected exactly one channel step {name!r}, found {len(matches)}") + step = matches[0] + if step.get("status") != "completed" or step.get("conclusion") != "success": + fail(f"required channel step {name!r} did not succeed") + +failed_channel_steps = [ + step.get("name", "") + for step in steps + if step.get("conclusion") == "failure" +] +if channels.get("conclusion") == "failure": + if failed_channel_steps != ["Sync release artifacts to China OSS mirror"]: + fail( + "failed channel publication must have exactly one failed OSS mirror step, " + f"got {failed_channel_steps!r}" + ) +elif failed_channel_steps: + fail(f"successful channel publication contains failed steps {failed_channel_steps!r}") + +gitee = one_job("Mirror immutable release to Gitee") +if gitee.get("conclusion") not in {"success", "skipped", "failure"}: + fail("Gitee mirror job did not complete with an allowed channel result") + +delivery_gate = one_job("Release delivery gate") +if delivery_gate.get("conclusion") != "failure": + fail("failed channel-repair run must end in a failed delivery gate") + +allowed_failures = { + "Publish npm and mirrors", + "Mirror immutable release to Gitee", + "Release delivery gate", +} +for job in jobs: + if job.get("status") != "completed": + fail(f"job {job.get('name', '')!r} is not completed") + conclusion = job.get("conclusion") + if conclusion not in {"success", "skipped", "failure"}: + fail(f"job {job.get('name', '')!r} has disallowed conclusion {conclusion!r}") + if conclusion == "failure" and job.get("name") not in allowed_failures: + fail(f"unrelated job {job.get('name', '')!r} failed") + +business_failures = [ + job.get("name") + for job in (channels, gitee) + if job.get("conclusion") == "failure" +] +if len(business_failures) != 1: + fail(f"expected exactly one failed downstream channel job, got {business_failures!r}") +if target == "oss": + if business_failures != ["Publish npm and mirrors"] or gitee.get("conclusion") != "skipped": + fail( + "OSS repair requires the OSS mirror step to be the only failed " + "downstream channel and Gitee to be skipped" + ) +elif target == "gitee": + if gitee.get("conclusion") == "failure": + if business_failures != ["Mirror immutable release to Gitee"]: + fail("Gitee repair evidence contains a different failed downstream channel") + elif gitee.get("conclusion") == "skipped": + if business_failures != ["Publish npm and mirrors"]: + fail( + "skipped Gitee backfill requires the upstream OSS mirror to be " + "the only failed downstream channel" + ) + else: + fail("Gitee repair requires its mirror job to be failed or skipped") +else: + fail(f"unsupported channel repair target {target!r}") +PY + rm -rf "$jobs_dir" + return "$result" +} + push_delivery="$(find_push_delivery || true)" if [ -n "$push_delivery" ]; then printf 'Release workflow delivery verified through exact-tag push run %s: %s -> %s\n' \ @@ -72,6 +282,18 @@ if [ -n "$push_delivery" ]; then exit 0 fi +if [ "$MODE" = "channel-repair" ]; then + failed_push_identity="$(find_failed_push_delivery || true)" + failed_push_delivery="$(printf '%s\n' "$failed_push_identity" | cut -f1)" + failed_push_attempt="$(printf '%s\n' "$failed_push_identity" | cut -f2)" + if [ -n "$failed_push_delivery" ] && + verify_channel_repair_delivery "$failed_push_delivery" "$failed_push_attempt"; then + printf 'Release %s channel-repair authority verified through failed exact-tag push run %s attempt %s: %s -> %s\n' \ + "$CHANNEL_REPAIR_TARGET" "$failed_push_delivery" "$failed_push_attempt" "$TAG" "$EXPECTED_COMMIT" + exit 0 + fi +fi + find_recovery_identity() { page=1 while :; do diff --git a/test/scripts/package_script_test.go b/test/scripts/package_script_test.go index 03303efc..7ea64d35 100644 --- a/test/scripts/package_script_test.go +++ b/test/scripts/package_script_test.go @@ -799,36 +799,49 @@ func TestReleaseWorkflowGovernancePreflightCannotPublish(t *testing.T) { } } -func TestReleaseWorkflowGiteeRepairUsesSealedReleaseAuthority(t *testing.T) { +func TestReleaseWorkflowChannelRepairUsesSealedReleaseAuthority(t *testing.T) { t.Parallel() workflow := readReleaseWorkflow(t) dispatch := releaseWorkflowSection(t, workflow, " dispatch-contract:\n", "\n authorize-recovery:\n") - start := strings.Index(workflow, " repair-gitee:\n") + start := strings.Index(workflow, " repair-channel:\n") if start == -1 { - t.Fatal("release workflow is missing the Gitee repair job") + t.Fatal("release workflow is missing the channel repair job") } repair := workflow[start:] + tagAuthority := releaseWorkflowSection( + t, + repair, + " - name: Fetch and verify sealed release tag\n", + "\n - name: Require successful Release workflow delivery\n", + ) for _, required := range []string{ "repair_gitee_version:", `format('Release Gitee repair {0}', inputs.repair_gitee_version)`, `REPAIR_GITEE_VERSION: ${{ inputs.repair_gitee_version }}`, + "repair_oss_version:", + `format('Release OSS repair {0}', inputs.repair_oss_version)`, + `REPAIR_OSS_VERSION: ${{ inputs.repair_oss_version }}`, "gitee_repair=0", + "oss_repair=0", `test -z "$REPAIR_GITEE_VERSION" || gitee_repair=1`, - "npm_repair + gitee_repair + governance + recovery", + `test -z "$REPAIR_OSS_VERSION" || oss_repair=1`, + "npm_repair + gitee_repair + oss_repair + governance + recovery", `echo "mode=repair_gitee" >> "$GITHUB_OUTPUT"`, + `echo "mode=repair_oss" >> "$GITHUB_OUTPUT"`, } { if !strings.Contains(dispatch, required) && !strings.Contains(workflow, required) { - t.Errorf("Gitee repair dispatch contract is missing %q", required) + t.Errorf("channel repair dispatch contract is missing %q", required) } } for _, required := range []string{ "needs: dispatch-contract", - `if: ${{ !cancelled() && needs.dispatch-contract.result == 'success' && needs.dispatch-contract.outputs.mode == 'repair_gitee'`, + `if: ${{ !cancelled() && needs.dispatch-contract.result == 'success' && (needs.dispatch-contract.outputs.mode == 'repair_gitee' || needs.dispatch-contract.outputs.mode == 'repair_oss') && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.repository == 'DingTalk-Real-AI/dingtalk-workspace-cli' }}`, `github.ref == format('refs/heads/{0}', github.event.repository.default_branch)`, `github.repository == 'DingTalk-Real-AI/dingtalk-workspace-cli'`, "actions: read", + "contents: read", `ref: ${{ github.sha }}`, "path: tooling", "persist-credentials: false", @@ -841,17 +854,34 @@ func TestReleaseWorkflowGiteeRepairUsesSealedReleaseAuthority(t *testing.T) { "assetNames.length !== expectedAssets.size", "new Set(assetNames).size !== expectedAssets.size", `core.setOutput("tag_object", ref.data.object.sha)`, + `ref: ${{ steps.authority.outputs.commit_sha }}`, + "path: release-source", "verify-github-tag-authority.sh", + `GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}`, "verify-release-workflow-delivery.sh", + `REPAIR_MODE: ${{ needs.dispatch-contract.outputs.mode }}`, + "repair_gitee) target=gitee", + "repair_oss) target=oss", + `--channel-repair "$target"`, `DWS_RELEASE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}`, "verify-release-artifacts.sh", `--repo "$GITHUB_REPOSITORY"`, + `if: ${{ needs.dispatch-contract.outputs.mode == 'repair_gitee' }}`, `DWS_REQUIRE_GITEE: "1"`, + `if: ${{ needs.dispatch-contract.outputs.mode == 'repair_oss' }}`, + `DWS_REQUIRE_OSS: "1"`, + `OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}`, + `OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}`, + `OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }}`, + `OSS_BUCKET: ${{ secrets.OSS_BUCKET }}`, + `OSS_PREFIX: ${{ secrets.OSS_PREFIX }}`, + "working-directory: release-source", "working-directory: tooling\n run: |\n " + `"$GITHUB_WORKSPACE/tooling/scripts/release/sync-to-gitee.sh"`, + `"$GITHUB_WORKSPACE/tooling/scripts/release/sync-to-oss.sh"`, } { if !strings.Contains(repair, required) { - t.Errorf("Gitee repair authority is missing %q", required) + t.Errorf("channel repair authority is missing %q", required) } } for _, asset := range []string{ @@ -865,17 +895,29 @@ func TestReleaseWorkflowGiteeRepairUsesSealedReleaseAuthority(t *testing.T) { "checksums.txt", } { if strings.Count(repair, `"`+asset+`"`) != 1 { - t.Errorf("Gitee repair must require exactly one %s asset declaration", asset) + t.Errorf("channel repair must require exactly one %s asset declaration", asset) } } if strings.Contains(repair, "contents: write") { - t.Error("Gitee repair must not grant contents write permission") + t.Error("channel repair must not grant contents write permission") + } + for _, forbidden := range []string{ + "RELEASE_GOVERNANCE_TOKEN", + "HOMEBREW_PR_TOKEN", + "NPM_TOKEN", + } { + if strings.Contains(repair, forbidden) { + t.Errorf("channel repair must not expose unrelated credential %s", forbidden) + } } if strings.Contains(repair, "ref: ${{ github.event.repository.default_branch }}") { - t.Error("Gitee repair must not check out a floating default branch") + t.Error("channel repair must not check out a floating default branch") + } + if !strings.Contains(tagAuthority, `GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}`) { + t.Error("remote annotated-tag authority step must receive the read-only GitHub token") } if got := strings.Count(repair, "working-directory: tooling"); got < 4 { - t.Errorf("Gitee repair trusted tooling working-directory count = %d, want at least 4", got) + t.Errorf("channel repair trusted tooling working-directory count = %d, want at least 4", got) } } @@ -1134,7 +1176,7 @@ func TestReleaseWorkflowPublicationBypassesSkippedDispatchButStopsOnCancellation func TestReleaseWorkflowDeliveryGateFailsClosed(t *testing.T) { t.Parallel() workflow := readReleaseWorkflow(t) - gate := releaseWorkflowSection(t, workflow, " release-delivery-gate:\n", "\n repair-gitee:\n") + gate := releaseWorkflowSection(t, workflow, " release-delivery-gate:\n", "\n repair-channel:\n") for _, required := range []string{ "name: Release delivery gate", @@ -1149,8 +1191,8 @@ func TestReleaseWorkflowDeliveryGateFailsClosed(t *testing.T) { "- publish-channels", "- mirror-gitee-release", "- repair-npm", - "- repair-gitee", - `REPAIR_GITEE_RESULT: ${{ needs.repair-gitee.result }}`, + "- repair-channel", + `REPAIR_CHANNEL_RESULT: ${{ needs.repair-channel.result }}`, "require_publication", `require_result release-contract "$RELEASE_CONTRACT_RESULT" success`, `require_result release "$RELEASE_RESULT" success`, @@ -1161,8 +1203,9 @@ func TestReleaseWorkflowDeliveryGateFailsClosed(t *testing.T) { "workflow_dispatch:governance_preflight", "workflow_dispatch:repair_npm", "workflow_dispatch:repair_gitee", - `require_result repair-gitee "$REPAIR_GITEE_RESULT" success`, - `require_result repair-gitee "$REPAIR_GITEE_RESULT" skipped`, + "workflow_dispatch:repair_oss", + `require_result repair-channel "$REPAIR_CHANNEL_RESULT" success`, + `require_result repair-channel "$REPAIR_CHANNEL_RESULT" skipped`, "unsupported release mode", } { if !strings.Contains(gate, required) { diff --git a/test/scripts/release_script_test.go b/test/scripts/release_script_test.go index fa799fa2..175ee3c0 100644 --- a/test/scripts/release_script_test.go +++ b/test/scripts/release_script_test.go @@ -674,6 +674,257 @@ esac } } +func TestReleaseWorkflowDeliveryChannelRepairRequiresLatestAttemptCoreDelivery(t *testing.T) { + sourceRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("Abs(repo root) error = %v", err) + } + binDir := t.TempDir() + fakeCurl := filepath.Join(binDir, "curl") + mustWriteFile(t, fakeCurl, []byte(`#!/bin/sh +set -eu +for argument in "$@"; do endpoint="$argument"; done +case "$endpoint" in + *event=push*) + python3 - <<'PY' +import json +import os +run = { + "id": 77, + "event": "push", + "status": "completed", + "conclusion": "failure", + "head_branch": os.environ["TAG"], + "head_sha": os.environ["RELEASE_COMMIT"], + "path": os.environ.get("RUN_PATH", ".github/workflows/release.yml"), + "repository": {"full_name": os.environ.get("RUN_REPOSITORY", "owner/repo")}, + "run_attempt": int(os.environ.get("RUN_ATTEMPT", "2")), +} +runs = [run] +if os.environ.get("DUPLICATE_RUN") == "1": + duplicate = dict(run) + duplicate["id"] = 78 + runs.append(duplicate) +print(json.dumps({"workflow_runs": runs})) +PY + ;; + *event=workflow_dispatch*) + printf '{"workflow_runs":[]}\n' + ;; + */actions/runs/77/attempts/2/jobs*) + python3 - <<'PY' +import json +import os + +commit = os.environ.get("JOB_SHA", os.environ["RELEASE_COMMIT"]) +core = [ + "release-contract", + "Build signed release artifacts", + "Verify Apple Developer ID signatures", + "Publish immutable GitHub Release", +] +jobs = [{ + "name": name, + "status": "completed", + "conclusion": ( + os.environ.get("CORE_CONCLUSION", "success") + if name == os.environ.get("CORE_JOB", "Build signed release artifacts") + else "success" + ), + "head_sha": commit, + "steps": ( + [{ + "name": "Require immutable published GitHub Release", + "status": "completed", + "conclusion": os.environ.get("IMMUTABLE_STEP_CONCLUSION", "success"), + }] + if name == "Publish immutable GitHub Release" + else [] + ), +} for name in core] +if os.environ.get("DUPLICATE_CORE") == "1": + jobs.append(dict(jobs[1])) +required_steps = [ + "Download and verify immutable GitHub Release", + "Verify immutable npm package without publication credentials", + "Inspect npm channel state", + "Verify npm channel delivery", +] +jobs.append({ + "name": "Publish npm and mirrors", + "status": "completed", + "conclusion": os.environ.get("CHANNEL_JOB_CONCLUSION", "failure"), + "head_sha": commit, + "steps": [{ + "name": name, + "status": "completed", + "conclusion": ( + os.environ.get("CHANNEL_STEP_CONCLUSION", "success") + if name == os.environ.get("CHANNEL_STEP", "Verify npm channel delivery") + else "success" + ), + } for name in required_steps] + [{ + "name": "Sync release artifacts to China OSS mirror", + "status": "completed", + "conclusion": os.environ.get("OSS_STEP_CONCLUSION", "failure"), + }], +}) +jobs.extend([ + { + "name": "Mirror immutable release to Gitee", + "status": "completed", + "conclusion": os.environ.get("GITEE_CONCLUSION", "skipped"), + "head_sha": commit, + "steps": [], + }, + { + "name": "Release delivery gate", + "status": "completed", + "conclusion": os.environ.get("DELIVERY_GATE_CONCLUSION", "failure"), + "head_sha": commit, + "steps": [], + }, +]) +if os.environ.get("UNRELATED_FAILURE") == "1": + jobs.append({ + "name": "Unrelated release job", + "status": "completed", + "conclusion": "failure", + "head_sha": commit, + "steps": [], + }) +print(json.dumps({"jobs": jobs})) +PY + ;; + */actions/runs/77/*/jobs*) + echo "channel repair must inspect the exact latest run attempt" >&2 + exit 91 + ;; + *) exit 1 ;; +esac +`), 0o755) + script := filepath.Join(sourceRoot, "scripts", "release", "verify-release-workflow-delivery.sh") + tag := "v1.2.3-beta.1" + commit := strings.Repeat("a", 40) + run := func(args []string, overrides ...string) (string, error) { + cmd := exec.Command("sh", append([]string{script}, args...)...) + cmd.Env = append([]string{ + "PATH=" + binDir + string(os.PathListSeparator) + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "DWS_RELEASE_OFFICIAL_REPOSITORY=owner/repo", + "TAG=" + tag, + "RELEASE_COMMIT=" + commit, + }, overrides...) + output, err := cmd.CombinedOutput() + return string(output), err + } + repairArgs := func(target string) []string { + return []string{"--channel-repair", target, tag, commit} + } + + if output, err := run([]string{tag, commit}); err == nil || + !strings.Contains(output, "did not deliver") { + t.Fatalf("strict delivery accepted a failed tag run: err=%v\noutput:\n%s", err, output) + } + if output, err := run(repairArgs("oss")); err != nil || + !strings.Contains(output, "failed exact-tag push run 77") { + t.Fatalf("safe channel repair delivery was rejected: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "CORE_CONCLUSION=failure", + ); err == nil || !strings.Contains(output, "required job 'Build signed release artifacts' did not succeed") { + t.Fatalf("failed core release job passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "CHANNEL_STEP_CONCLUSION=failure", + ); err == nil || !strings.Contains(output, "required channel step 'Verify npm channel delivery' did not succeed") { + t.Fatalf("failed npm delivery proof passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "DUPLICATE_CORE=1", + ); err == nil || !strings.Contains(output, "expected exactly one latest-attempt job 'Build signed release artifacts'") { + t.Fatalf("duplicate latest-attempt core job passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "JOB_SHA="+strings.Repeat("b", 40), + ); err == nil || !strings.Contains(output, "is not bound to "+commit) { + t.Fatalf("wrong-sha release jobs passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "UNRELATED_FAILURE=1", + ); err == nil || !strings.Contains(output, "unrelated job 'Unrelated release job' failed") { + t.Fatalf("unrelated failed job passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "IMMUTABLE_STEP_CONCLUSION=failure", + ); err == nil || !strings.Contains(output, "immutable GitHub Release verification did not succeed") { + t.Fatalf("failed immutable release verification passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("gitee"), + "CHANNEL_JOB_CONCLUSION=success", + "OSS_STEP_CONCLUSION=success", + "GITEE_CONCLUSION=failure", + ); err != nil || !strings.Contains(output, "channel-repair authority verified") { + t.Fatalf("single failed Gitee mirror was rejected: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "CHANNEL_JOB_CONCLUSION=success", + "OSS_STEP_CONCLUSION=success", + "GITEE_CONCLUSION=failure", + ); err == nil || !strings.Contains(output, "OSS repair requires") { + t.Fatalf("Gitee-only failure was accepted as OSS evidence: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("gitee"), + ); err != nil || !strings.Contains(output, "channel-repair authority verified") { + t.Fatalf("skipped Gitee backfill was rejected after upstream OSS failure: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "GITEE_CONCLUSION=failure", + ); err == nil || !strings.Contains(output, "expected exactly one failed downstream channel job") { + t.Fatalf("two failed downstream channels passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "DELIVERY_GATE_CONCLUSION=success", + ); err == nil || !strings.Contains(output, "must end in a failed delivery gate") { + t.Fatalf("successful terminal gate on a failed run passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "RUN_ATTEMPT=1", + ); err == nil || strings.Contains(output, "channel-repair authority verified") { + t.Fatalf("non-latest run attempt passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "RUN_REPOSITORY=other/repo", + ); err == nil || strings.Contains(output, "channel-repair authority verified") { + t.Fatalf("wrong-repository release run passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "RUN_PATH=.github/workflows/other.yml", + ); err == nil || strings.Contains(output, "channel-repair authority verified") { + t.Fatalf("wrong-workflow release run passed: err=%v\noutput:\n%s", err, output) + } + if output, err := run( + repairArgs("oss"), + "DUPLICATE_RUN=1", + ); err == nil || strings.Contains(output, "channel-repair authority verified") { + t.Fatalf("ambiguous failed release runs passed: err=%v\noutput:\n%s", err, output) + } +} + func releaseChangelog(sections ...string) string { text := "# Changelog\n\n## [Unreleased]\n\n" for _, section := range sections {