diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7147bf5a..13cc8fbc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1735,7 +1735,7 @@ jobs: if: ${{ !cancelled() && vars.ENABLE_GITEE_UPLOAD_FALLBACK == 'true' && needs.release-contract.result == 'success' && needs.release.result == 'success' && needs.publish-channels.result == 'success' }} needs: [release-contract, release, publish-channels] runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 360 permissions: contents: read env: @@ -1752,12 +1752,14 @@ jobs: - name: Check out trusted release tooling uses: actions/checkout@v4 + timeout-minutes: 5 with: ref: ${{ github.sha }} path: tmp/trusted-release-tooling persist-credentials: false - name: Fetch and verify sealed release tag + timeout-minutes: 5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -1776,7 +1778,7 @@ jobs: path: dist - name: Mirror release to Gitee (China) - timeout-minutes: 100 + timeout-minutes: 318 run: ./scripts/release/sync-to-gitee.sh env: VERSION: ${{ needs.release-contract.outputs.release_version }} @@ -2180,13 +2182,14 @@ jobs: needs: dispatch-contract if: ${{ !cancelled() && needs.dispatch-contract.result == 'success' && (needs.dispatch-contract.outputs.mode == 'repair_gitee' || needs.dispatch-contract.outputs.mode == 'repair_oss') && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.repository == 'DingTalk-Real-AI/dingtalk-workspace-cli' }} runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 360 permissions: actions: read contents: read steps: - name: Check out trusted release tooling uses: actions/checkout@v4 + timeout-minutes: 5 with: ref: ${{ github.sha }} path: tooling @@ -2194,6 +2197,7 @@ jobs: fetch-depth: 0 - name: Validate repair version + timeout-minutes: 2 working-directory: tooling env: VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} @@ -2208,6 +2212,7 @@ jobs: - name: Verify immutable release authority id: authority uses: actions/github-script@v7 + timeout-minutes: 5 env: VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} with: @@ -2330,12 +2335,14 @@ jobs: - name: Check out sealed release source uses: actions/checkout@v4 + timeout-minutes: 5 with: ref: ${{ steps.authority.outputs.commit_sha }} path: release-source persist-credentials: false - name: Fetch and verify sealed release tag + timeout-minutes: 5 working-directory: tooling env: VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} @@ -2352,6 +2359,7 @@ jobs: "$VERSION" "$RELEASE_COMMIT" "$RELEASE_TAG_OBJECT" - name: Require successful Release workflow delivery + timeout-minutes: 5 working-directory: tooling env: VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} @@ -2368,6 +2376,7 @@ jobs: --channel-repair "$target" "$VERSION" "$RELEASE_COMMIT" - name: Download and verify immutable GitHub Release assets + timeout-minutes: 10 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }} @@ -2385,7 +2394,7 @@ jobs: - name: Mirror release to Gitee (China) if: ${{ needs.dispatch-contract.outputs.mode == 'repair_gitee' }} - timeout-minutes: 100 + timeout-minutes: 318 working-directory: tooling run: | "$GITHUB_WORKSPACE/tooling/scripts/release/sync-to-gitee.sh" diff --git a/Formula/dingtalk-workspace-cli-beta.rb b/Formula/dingtalk-workspace-cli-beta.rb index 8076eb26..8deca37e 100644 --- a/Formula/dingtalk-workspace-cli-beta.rb +++ b/Formula/dingtalk-workspace-cli-beta.rb @@ -1,33 +1,33 @@ class DingtalkWorkspaceCliBeta < Formula desc "Automate DingTalk workspace tasks from the terminal (beta channel)" homepage "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli" - version "1.0.53-beta.4" + version "1.0.54-beta.2" license "Apache-2.0" keg_only "it is the beta channel and conflicts with dingtalk-workspace-cli" on_macos do if Hardware::CPU.arm? - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.53-beta.4/dws-darwin-arm64.tar.gz" - sha256 "32a442d5b42dfed8512a695a7cef513722db6912f3c3168954cfaefb68e0b075" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54-beta.2/dws-darwin-arm64.tar.gz" + sha256 "46b57bed1f6e9f7ba007d8a86a6f5eb280fdeb557fc9bb5946f14f9b1f8f0c9f" else - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.53-beta.4/dws-darwin-amd64.tar.gz" - sha256 "00c694677b9ce2e1a711535740681defe0a5f83d6b72140f305483501628faf8" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54-beta.2/dws-darwin-amd64.tar.gz" + sha256 "1b7fd08e64b1c86bbcee217604ffe07e0e8f1b3b5c4de518534386972bcf0f9b" end end on_linux do if Hardware::CPU.arm? - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.53-beta.4/dws-linux-arm64.tar.gz" - sha256 "98516620e861e516cf846cf418f1a0ad5c5eb9a8681bd066b1fd29f4847aca6a" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54-beta.2/dws-linux-arm64.tar.gz" + sha256 "108d3861ef606519f9934530d29654eab55a73607d1ee6775461f98ef5a6acd4" else - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.53-beta.4/dws-linux-amd64.tar.gz" - sha256 "266df80e8a989971789a157dd134685a7c9eda01dd1d082719ec9e09d5a07bf0" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54-beta.2/dws-linux-amd64.tar.gz" + sha256 "6cb96ee09419bbbcc1eb336218ac2aa1d9ca0ed5cbd5a80c79bc20e1e1f03ff7" end end resource "skills" do - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.53-beta.4/dws-skills.zip" - sha256 "6770511ab9b04b4d97da1858069ff694830ba91d47c1e8564fd85856f95b016e" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54-beta.2/dws-skills.zip" + sha256 "572b93f04a10268d185ad1f8e70e0d412949ae056be8494a9949387076fd14bc" end def install diff --git a/Formula/dingtalk-workspace-cli.rb b/Formula/dingtalk-workspace-cli.rb index 0416b9a1..c6846a01 100644 --- a/Formula/dingtalk-workspace-cli.rb +++ b/Formula/dingtalk-workspace-cli.rb @@ -1,32 +1,33 @@ class DingtalkWorkspaceCli < Formula desc "Automate DingTalk workspace tasks from the terminal" homepage "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli" - version "1.0.52" + version "1.0.54" license "Apache-2.0" + on_macos do if Hardware::CPU.arm? - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.52/dws-darwin-arm64.tar.gz" - sha256 "4f6b4d064a76bcefac42feb5f356253fe43f9499b8cec9d2cdf202e7d3b9b60c" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54/dws-darwin-arm64.tar.gz" + sha256 "8ae0e52cf973f6fb3df61c67a41fd11e2df417a0c815762b6060cbcb5e600c08" else - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.52/dws-darwin-amd64.tar.gz" - sha256 "abc87128f4b98d0a01ea99235449031971db8fa4ce94167403e3b736c4b81e9a" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54/dws-darwin-amd64.tar.gz" + sha256 "11b711b9d70dea62304bf5f8206c56b4e7ea91148dafe97fb7c0f844a2a61da3" end end on_linux do if Hardware::CPU.arm? - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.52/dws-linux-arm64.tar.gz" - sha256 "0d357ef0535f99f2f63b5ecbfdee9c32448be2a2c24f3096c03126b3b7570bc5" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54/dws-linux-arm64.tar.gz" + sha256 "9c7ecb4c8cd55644b2faa73f6ce7843c0279b23793e23deb5061692ea71a0cf1" else - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.52/dws-linux-amd64.tar.gz" - sha256 "b7dfd9a4b3489211359261747ed0cb9c8c261434bb762ad3f76df33bdbabd5cb" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54/dws-linux-amd64.tar.gz" + sha256 "8a0bc245747fc3facf98c8103c06da46852a30bff31ac93b0aa874e8c7e46db7" end end resource "skills" do - url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.52/dws-skills.zip" - sha256 "0fa3c8dec500c1659e6480d6772ae901b2d12d24322dd5d7283f016024290c21" + url "https://github.com/DingTalk-Real-AI/dingtalk-workspace-cli/releases/download/v1.0.54/dws-skills.zip" + sha256 "7450fd0115c75bfe6820c7099f348973d9353cca9d8d647c9cddcd70978a7ec0" end def install @@ -52,6 +53,7 @@ class DingtalkWorkspaceCli < Formula <<~EOS Agent Skills are bundled in #{pkgshare}/skills/dws. Run `dws skill setup` to install them into your Agent directories. + EOS end diff --git a/scripts/release/reconcile-gitee-assets.sh b/scripts/release/reconcile-gitee-assets.sh index 5bf443b6..50ce4ace 100755 --- a/scripts/release/reconcile-gitee-assets.sh +++ b/scripts/release/reconcile-gitee-assets.sh @@ -8,16 +8,32 @@ GITEE_API="${GITEE_API:-https://gitee.com/api/v5}" GITEE_CURL_CONNECT_TIMEOUT="${GITEE_CURL_CONNECT_TIMEOUT:-15}" GITEE_CURL_MAX_TIME="${GITEE_CURL_MAX_TIME:-120}" GITEE_LIST_MAX_TIME="${GITEE_LIST_MAX_TIME:-20}" +# Attachment-list reads are safe to retry and are used both before an upload +# and to detect a committed upload whose HTTP response was lost. Keep retrying +# long enough to bridge a short Gitee TLS/API outage without ever blindly +# replaying the upload itself. +GITEE_LIST_RETRIES="${GITEE_LIST_RETRIES:-24}" +GITEE_LIST_RETRY_DELAY="${GITEE_LIST_RETRY_DELAY:-20}" +GITEE_LIST_RETRY_WINDOW_SECONDS="${GITEE_LIST_RETRY_WINDOW_SECONDS:-420}" GITEE_VERIFY_MAX_TIME="${GITEE_VERIFY_MAX_TIME:-60}" GITEE_MUTATION_MAX_TIME="${GITEE_MUTATION_MAX_TIME:-20}" -GITEE_UPLOAD_MAX_TIME="${GITEE_UPLOAD_MAX_TIME:-120}" +# Gitee does not return a response until an attachment upload is committed. +# From GitHub-hosted runners, a near-10 MiB DWS binary can legitimately take +# more than five minutes, so keep the transfer deadline above that observed +# floor while the per-asset and overall deadlines retain hard upper bounds. +GITEE_UPLOAD_MAX_TIME="${GITEE_UPLOAD_MAX_TIME:-1200}" +# The per-asset deadline guarantees one complete slow upload plus one complete +# attachment-list recovery on each side of that upload. The second upload +# attempt is allowed only for zero-byte failures before HTTP begins; a first +# attempt that consumes the full transfer deadline intentionally exhausts the +# retry budget instead of holding a runner for another full slow upload. GITEE_UPLOAD_RETRIES="${GITEE_UPLOAD_RETRIES:-2}" GITEE_UPLOAD_RETRY_DELAY="${GITEE_UPLOAD_RETRY_DELAY:-5}" GITEE_EXISTING_VERIFY_ATTEMPTS="${GITEE_EXISTING_VERIFY_ATTEMPTS:-1}" GITEE_POST_UPLOAD_VERIFY_ATTEMPTS="${GITEE_POST_UPLOAD_VERIFY_ATTEMPTS:-2}" GITEE_VERIFY_RETRY_DELAY="${GITEE_VERIFY_RETRY_DELAY:-5}" -GITEE_ASSET_TIMEOUT_SECONDS="${GITEE_ASSET_TIMEOUT_SECONDS:-600}" -GITEE_OVERALL_TIMEOUT_SECONDS="${GITEE_OVERALL_TIMEOUT_SECONDS:-4920}" +GITEE_ASSET_TIMEOUT_SECONDS="${GITEE_ASSET_TIMEOUT_SECONDS:-2220}" +GITEE_OVERALL_TIMEOUT_SECONDS="${GITEE_OVERALL_TIMEOUT_SECONDS:-18300}" err() { printf 'error: %s\n' "$*" >&2 @@ -43,6 +59,8 @@ for setting in \ GITEE_CURL_CONNECT_TIMEOUT \ GITEE_CURL_MAX_TIME \ GITEE_LIST_MAX_TIME \ + GITEE_LIST_RETRIES \ + GITEE_LIST_RETRY_WINDOW_SECONDS \ GITEE_VERIFY_MAX_TIME \ GITEE_MUTATION_MAX_TIME \ GITEE_UPLOAD_MAX_TIME \ @@ -54,6 +72,7 @@ for setting in \ require_positive_integer "$setting" "${!setting}" done for setting in \ + GITEE_LIST_RETRY_DELAY \ GITEE_UPLOAD_RETRY_DELAY \ GITEE_VERIFY_RETRY_DELAY; do require_nonnegative_integer "$setting" "${!setting}" @@ -139,21 +158,81 @@ api_get() { --max-time "$max_time" "$@" } -list_assets() { - api_get "$GITEE_LIST_MAX_TIME" \ +list_assets_once() { + local max_time="$1" payload + payload="$(api_get "$max_time" \ -H "Authorization: token ${GITEE_TOKEN}" \ - "${base}/releases/${release_id}/attach_files" \ - | python3 -c 'import json,sys + "${base}/releases/${release_id}/attach_files")" || return 1 + printf '%s\n' "$payload" | python3 -c 'import json,sys data=json.load(sys.stdin) -rows=data if isinstance(data,list) else data.get("attach_files",[]) +if isinstance(data,list): + rows=data +elif isinstance(data,dict) and "attach_files" in data: + rows=data["attach_files"] +else: + raise ValueError("attachment list response must be a list or contain attach_files") if not isinstance(rows,list): raise ValueError("attach_files must be a list") for asset in rows: + if not isinstance(asset,dict): + raise ValueError("each attachment must be an object") name=asset.get("name","") asset_id=asset.get("id","") url=asset.get("browser_download_url","") - if name and asset_id != "": - print("%s\t%s\t%s" % (name, asset_id, url))' + valid_id=(isinstance(asset_id,int) and not isinstance(asset_id,bool) and asset_id > 0) or (isinstance(asset_id,str) and asset_id.isdigit() and int(asset_id) > 0) + if not isinstance(name,str) or not name or not valid_id or not isinstance(url,str) or not url: + raise ValueError("attachment identity and download URL are required") + print("%s\t%s\t%s" % (name, asset_id, url))' +} + +list_assets() { + local attempt=1 candidate now retry_deadline remaining max_time delay + now="$(now_seconds)" + retry_deadline=$(( now + GITEE_LIST_RETRY_WINDOW_SECONDS )) + if [ "$retry_deadline" -gt "$active_deadline" ]; then + retry_deadline="$active_deadline" + fi + while [ "$attempt" -le "$GITEE_LIST_RETRIES" ]; do + now="$(now_seconds)" + remaining=$(( retry_deadline - now )) + [ "$remaining" -gt 0 ] || break + max_time="$GITEE_LIST_MAX_TIME" + if [ "$max_time" -gt "$remaining" ]; then + max_time="$remaining" + fi + # Publish one complete parse atomically. A malformed response can make the + # parser emit valid leading rows before it fails; leaking those rows into a + # later successful retry would manufacture duplicates and could delete a + # correct attachment. + if candidate="$(list_assets_once "$max_time")"; then + now="$(now_seconds)" + if [ "$now" -lt "$retry_deadline" ]; then + printf '%s\n' "$candidate" + return 0 + fi + break + fi + if [ "$attempt" -ge "$GITEE_LIST_RETRIES" ]; then + break + fi + attempt=$((attempt + 1)) + now="$(now_seconds)" + remaining=$(( retry_deadline - now )) + [ "$remaining" -gt 0 ] || break + delay="$GITEE_LIST_RETRY_DELAY" + if [ "$delay" -gt 0 ]; then + # Do not turn a configured backoff into a burst of zero-delay requests + # during the final wall-clock second. Explicit delay=0 remains available + # to fast unit tests and tightly controlled callers. + [ "$remaining" -gt 1 ] || break + if [ "$delay" -ge "$remaining" ]; then + delay=$(( remaining - 1 )) + fi + fi + echo " ⚠ Gitee attachment list attempt $((attempt - 1))/${GITEE_LIST_RETRIES} failed; retrying in ${delay}s" >&2 + sleep_within_deadline "$delay" || return 1 + done + return 1 } sha256_of() { @@ -234,20 +313,42 @@ delete_named_assets() { } gitee_attach() { - local file="$1" name attempt response status max_time + local file="$1" name attempt response status max_time transfer_log metrics http_code size_upload safe_to_retry name="$(basename "$file")" attempt=1 while [ "$attempt" -le "$GITEE_UPLOAD_RETRIES" ]; do status=0 max_time="$(bounded_max_time "$GITEE_UPLOAD_MAX_TIME")" || return 1 - if response="$(curl -fsS --connect-timeout "$GITEE_CURL_CONNECT_TIMEOUT" \ + transfer_log="$(mktemp)" || return 1 + if metrics="$(curl -fsS --connect-timeout "$GITEE_CURL_CONNECT_TIMEOUT" \ --max-time "$max_time" \ -X POST "${base}/releases/${release_id}/attach_files" \ - -H "Authorization: token ${GITEE_TOKEN}" -F "file=@${file}" 2>&1)"; then + -H "Authorization: token ${GITEE_TOKEN}" \ + -H "Expect:" \ + -F "file=@${file}" \ + -o /dev/null \ + -w '%{http_code}\t%{size_upload}' \ + 2>"$transfer_log")"; then status=0 else status=$? fi + response="$(<"$transfer_log")" + rm -f "$transfer_log" + http_code="${metrics%%$'\t'*}" + if [ "$metrics" = "$http_code" ]; then + size_upload="" + else + size_upload="${metrics#*$'\t'}" + fi + safe_to_retry=0 + if [ "$status" -ne 0 ] && [ "$http_code" = "000" ] && \ + awk -v value="$size_upload" 'BEGIN { exit !((value + 0) == 0 && value ~ /^[0-9]+([.][0-9]+)?$/) }'; then + # A request that never reached HTTP and uploaded zero bytes cannot have + # committed an attachment. TLS/DNS/connect failures are therefore the + # only transport failures safe to replay automatically. + safe_to_retry=1 + fi # Gitee can commit an upload but let the HTTP response time out. Probe the # release before retrying so a lost response does not create duplicates. @@ -259,6 +360,10 @@ gitee_attach() { fi echo " ⚠ upload attempt ${attempt}/${GITEE_UPLOAD_RETRIES} failed for ${name}: $(printf '%s' "$response" | head -c 240)" >&2 + if [ "$safe_to_retry" -ne 1 ]; then + echo " ⚠ ${name} upload outcome is ambiguous (HTTP ${http_code:-unknown}, uploaded ${size_upload:-unknown} bytes); refusing to replay POST" >&2 + return 1 + fi attempt=$((attempt + 1)) if [ "$attempt" -le "$GITEE_UPLOAD_RETRIES" ]; then # Remove any partial, stale, or duplicate attachment before the one diff --git a/scripts/release/sync-to-gitee.sh b/scripts/release/sync-to-gitee.sh index 87c2965d..ad903589 100755 --- a/scripts/release/sync-to-gitee.sh +++ b/scripts/release/sync-to-gitee.sh @@ -31,11 +31,13 @@ DIST_DIR="${DIST_DIR:-dist}" GITEE_API="${GITEE_API:-https://gitee.com/api/v5}" GITEE_CURL_CONNECT_TIMEOUT="${GITEE_CURL_CONNECT_TIMEOUT:-15}" GITEE_CURL_MAX_TIME="${GITEE_CURL_MAX_TIME:-120}" -GITEE_SYNC_TIMEOUT_SECONDS="${GITEE_SYNC_TIMEOUT_SECONDS:-5700}" +GITEE_SYNC_TIMEOUT_SECONDS="${GITEE_SYNC_TIMEOUT_SECONDS:-18840}" GITEE_TAG_TIMEOUT_SECONDS="${GITEE_TAG_TIMEOUT_SECONDS:-300}" GITEE_RELEASE_LOOKUP_MAX_TIME="${GITEE_RELEASE_LOOKUP_MAX_TIME:-60}" +GITEE_RELEASE_LOOKUP_RETRIES="${GITEE_RELEASE_LOOKUP_RETRIES:-2}" +GITEE_RELEASE_LOOKUP_RETRY_DELAY="${GITEE_RELEASE_LOOKUP_RETRY_DELAY:-2}" GITEE_RELEASE_CREATE_MAX_TIME="${GITEE_RELEASE_CREATE_MAX_TIME:-60}" -GITEE_RECONCILE_TIMEOUT_SECONDS="${GITEE_RECONCILE_TIMEOUT_SECONDS:-4920}" +GITEE_RECONCILE_TIMEOUT_SECONDS="${GITEE_RECONCILE_TIMEOUT_SECONDS:-18300}" GITEE_CHILD_DEADLINE_RESERVE_SECONDS="${GITEE_CHILD_DEADLINE_RESERVE_SECONDS:-5}" err() { @@ -51,17 +53,28 @@ require_positive_integer() { [ "$value" -gt 0 ] || err "${name} must be greater than zero" } +require_nonnegative_integer() { + local name="$1" value="$2" + case "$value" in + ''|*[!0-9]*) err "${name} must be a non-negative integer: ${value}" ;; + esac +} + for setting in \ GITEE_CURL_CONNECT_TIMEOUT \ GITEE_CURL_MAX_TIME \ GITEE_SYNC_TIMEOUT_SECONDS \ GITEE_TAG_TIMEOUT_SECONDS \ GITEE_RELEASE_LOOKUP_MAX_TIME \ + GITEE_RELEASE_LOOKUP_RETRIES \ GITEE_RELEASE_CREATE_MAX_TIME \ GITEE_RECONCILE_TIMEOUT_SECONDS \ GITEE_CHILD_DEADLINE_RESERVE_SECONDS; do require_positive_integer "$setting" "${!setting}" done +require_nonnegative_integer \ + GITEE_RELEASE_LOOKUP_RETRY_DELAY \ + "$GITEE_RELEASE_LOOKUP_RETRY_DELAY" missing="" [ -z "${GITEE_TOKEN:-}" ] && missing="$missing GITEE_TOKEN" @@ -117,14 +130,6 @@ GITEE_TAG_TIMEOUT_SECONDS="$GITEE_TAG_TIMEOUT_SECONDS" \ deadline_remaining >/dev/null || err "overall Gitee sync deadline exhausted during tag synchronization" target_commit="$(git rev-parse --verify "${VERSION}^{commit}")" -api_get() { - local configured_max_time="$1" max_time - shift - max_time="$(bounded_max_time "$configured_max_time")" || return 1 - curl -fsSL --connect-timeout "$GITEE_CURL_CONNECT_TIMEOUT" \ - --max-time "$max_time" "$@" -} - release_id_from_json() { python3 -c 'import json, sys data = json.load(sys.stdin) @@ -136,12 +141,41 @@ elif isinstance(value, str) and value.isdigit() and int(value) > 0: } # ── Resolve or create the Gitee release for this tag ────────────────────────── -rel_json="$(api_get "$GITEE_RELEASE_LOOKUP_MAX_TIME" \ +# A transient lookup failure must not be mistaken for a missing release: doing +# so can race a duplicate create and hides the actual Gitee availability issue. +release_lookup_body="$(mktemp)" +trap 'rm -f "$release_lookup_body"' EXIT HUP INT TERM +lookup_max_time="$(bounded_max_time "$GITEE_RELEASE_LOOKUP_MAX_TIME")" \ + || err "overall Gitee sync deadline exhausted before release lookup" +if ! release_status="$(curl -sS --connect-timeout "$GITEE_CURL_CONNECT_TIMEOUT" \ + --max-time "$lookup_max_time" \ + --retry "$GITEE_RELEASE_LOOKUP_RETRIES" \ + --retry-all-errors \ + --retry-delay "$GITEE_RELEASE_LOOKUP_RETRY_DELAY" \ + --retry-max-time "$lookup_max_time" \ + -o "$release_lookup_body" \ + -w '%{http_code}' \ -H "Authorization: token ${GITEE_TOKEN}" \ - "${base}/releases/tags/${VERSION}" 2>/dev/null || true)" -release_id="$(printf '%s' "$rel_json" | release_id_from_json || true)" + "${base}/releases/tags/${VERSION}")"; then + deadline_remaining >/dev/null \ + || err "overall Gitee sync deadline exhausted during release lookup" + err "could not query Gitee release ${VERSION} after bounded retries" +fi +rel_json="$(<"$release_lookup_body")" +case "$release_status" in + 200) + release_id="$(printf '%s' "$rel_json" | release_id_from_json || true)" + [ -n "$release_id" ] || err "Gitee release lookup returned HTTP 200 without a valid release id" + ;; + 404) + release_id="" + ;; + *) + err "Gitee release lookup returned HTTP ${release_status} for ${VERSION}" + ;; +esac -if [ -z "$release_id" ]; then +if [ "$release_status" = "404" ]; then deadline_remaining >/dev/null || err "overall Gitee sync deadline exhausted during release lookup" echo " No Gitee release for ${VERSION} yet — creating it." create_max_time="$(bounded_max_time "$GITEE_RELEASE_CREATE_MAX_TIME")" \ diff --git a/test/scripts/gitee_sync_test.go b/test/scripts/gitee_sync_test.go index 9a0b35f5..bb8e715e 100644 --- a/test/scripts/gitee_sync_test.go +++ b/test/scripts/gitee_sync_test.go @@ -300,7 +300,12 @@ func TestSyncToGiteeRunsTagReleaseCreationAndAssetReconciliationWithinOneBudget( case r.Method == http.MethodGet && r.URL.Path == "/repos/owner/repo/releases/tags/v1.2.3": releaseMu.Lock() releaseLookups++ + lookupAttempt := releaseLookups releaseMu.Unlock() + if lookupAttempt == 1 { + http.Error(w, "temporary Gitee outage", http.StatusServiceUnavailable) + return + } http.NotFound(w, r) case r.Method == http.MethodPost && r.URL.Path == "/repos/owner/repo/releases": if err := r.ParseMultipartForm(1 << 20); err != nil { @@ -336,6 +341,8 @@ func TestSyncToGiteeRunsTagReleaseCreationAndAssetReconciliationWithinOneBudget( "GITEE_TAG_TIMEOUT_SECONDS=5", "GITEE_GIT_TIMEOUT_SECONDS=3", "GITEE_RELEASE_LOOKUP_MAX_TIME=2", + "GITEE_RELEASE_LOOKUP_RETRIES=2", + "GITEE_RELEASE_LOOKUP_RETRY_DELAY=0", "GITEE_RELEASE_CREATE_MAX_TIME=2", "GITEE_RECONCILE_TIMEOUT_SECONDS=20", "GITEE_CHILD_DEADLINE_RESERVE_SECONDS=1", @@ -357,8 +364,8 @@ func TestSyncToGiteeRunsTagReleaseCreationAndAssetReconciliationWithinOneBudget( } releaseMu.Lock() - if releaseLookups != 1 || releaseCreates != 1 { - t.Errorf("release lookup/create calls = %d/%d, want 1/1", releaseLookups, releaseCreates) + if releaseLookups != 2 || releaseCreates != 1 { + t.Errorf("release lookup/create calls = %d/%d, want 2/1", releaseLookups, releaseCreates) } releaseMu.Unlock() fake.mu.Lock() @@ -374,11 +381,15 @@ func TestReconcileGiteeAssetsRecoversACommittedUploadWithLostResponse(t *testing scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) distDir := seedGiteeDist(t) fake := newFakeGiteeRelease(true, false) + fake.listErrorResponsesAfterUpload = 1 + fake.listEmptyResponsesAfterUpload = 1 server := httptest.NewServer(fake) defer server.Close() cmd := exec.Command("bash", scriptPath) - cmd.Env = giteeAssetEnv(distDir, server.URL, "2") + cmd.Env = append(giteeAssetEnv(distDir, server.URL, "2"), + "GITEE_POST_UPLOAD_VERIFY_ATTEMPTS=2", + ) output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("reconcile-gitee-assets.sh error = %v\noutput:\n%s", err, output) @@ -409,6 +420,167 @@ func TestReconcileGiteeAssetsRecoversACommittedUploadWithLostResponse(t *testing } } +func TestReconcileGiteeAssetsDoesNotReplayAnAmbiguousUpload(t *testing.T) { + scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) + distDir := seedGiteeDist(t) + fake := newFakeGiteeRelease(true, false) + fake.listEmptyResponsesAfterUpload = 3 + server := httptest.NewServer(fake) + defer server.Close() + + cmd := exec.Command("bash", scriptPath) + cmd.Env = append(giteeAssetEnv(distDir, server.URL, "2"), + "GITEE_POST_UPLOAD_VERIFY_ATTEMPTS=2", + ) + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("ambiguous upload unexpectedly succeeded:\n%s", output) + } + if !strings.Contains(string(output), "refusing to replay POST") { + t.Fatalf("ambiguous upload did not fail closed:\n%s", output) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + for _, name := range requiredGiteeAssets { + if got := fake.uploadCalls[name]; got != 1 { + t.Errorf("upload calls for %s = %d, want exactly 1", name, got) + } + } +} + +func TestReconcileGiteeAssetsRetriesATransientListOutage(t *testing.T) { + scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) + distDir := seedGiteeDist(t) + fake := newFakeGiteeRelease(false, false) + fake.listFailuresRemaining = 2 + server := httptest.NewServer(fake) + defer server.Close() + + cmd := exec.Command("bash", scriptPath) + cmd.Env = append(giteeAssetEnv(distDir, server.URL, "1"), + "GITEE_LIST_RETRIES=3", + "GITEE_LIST_RETRY_DELAY=1", + "GITEE_LIST_RETRY_WINDOW_SECONDS=4", + ) + started := time.Now() + output, err := cmd.CombinedOutput() + elapsed := time.Since(started) + if err != nil { + t.Fatalf("transient-list reconciliation error = %v\noutput:\n%s", err, output) + } + if !strings.Contains(string(output), "Gitee attachment list attempt 2/3 failed; retrying in 1s") { + t.Fatalf("transient list failures were not retried visibly:\n%s", output) + } + if elapsed < 1500*time.Millisecond { + t.Fatalf("transient list retry elapsed = %s, want production-style backoff", elapsed) + } + if !strings.Contains(string(output), "all 8 verified") { + t.Fatalf("transient-list reconciliation did not verify every asset:\n%s", output) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.listFailuresRemaining != 0 { + t.Fatalf("unconsumed list failures = %d, want 0", fake.listFailuresRemaining) + } + for _, name := range requiredGiteeAssets { + if got := fake.uploadCalls[name]; got != 1 { + t.Errorf("upload calls for %s = %d, want 1", name, got) + } + } +} + +func TestReconcileGiteeAssetsDoesNotBurstRetriesInTheFinalWindowSecond(t *testing.T) { + scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) + distDir := seedGiteeDist(t) + fake := newFakeGiteeRelease(false, false) + fake.listFailuresRemaining = 1000 + server := httptest.NewServer(fake) + defer server.Close() + + cmd := exec.Command("bash", scriptPath) + cmd.Env = append(giteeAssetEnv(distDir, server.URL, "1"), + "GITEE_LIST_RETRIES=24", + "GITEE_LIST_RETRY_DELAY=1", + "GITEE_LIST_RETRY_WINDOW_SECONDS=1", + ) + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("permanent list outage unexpectedly succeeded:\n%s", output) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.listCalls > len(requiredGiteeAssets)+1 { + t.Fatalf("list calls = %d, want at most one per asset plus final verification", fake.listCalls) + } +} + +func TestReconcileGiteeAssetsDiscardsPartialOutputFromAMalformedList(t *testing.T) { + scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) + distDir := seedGiteeDist(t) + fake := newFakeGiteeRelease(false, false) + existingName := requiredGiteeAssets[0] + existingData, err := os.ReadFile(filepath.Join(distDir, existingName)) + if err != nil { + t.Fatalf("ReadFile(%s) error = %v", existingName, err) + } + fake.assets[1] = fakeGiteeAsset{id: 1, name: existingName, data: existingData} + fake.nextID = 2 + fake.malformedListResponsesRemaining = 1 + server := httptest.NewServer(fake) + defer server.Close() + + cmd := exec.Command("bash", scriptPath) + cmd.Env = append(giteeAssetEnv(distDir, server.URL, "1"), + "GITEE_LIST_RETRIES=2", + "GITEE_LIST_RETRY_DELAY=0", + "GITEE_LIST_RETRY_WINDOW_SECONDS=2", + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("malformed-list reconciliation error = %v\noutput:\n%s", err, output) + } + if !strings.Contains(string(output), existingName+" already correct on Gitee") { + t.Fatalf("partial failed-list output manufactured a stale or duplicate asset:\n%s", output) + } + if !strings.Contains(string(output), "all 8 verified") { + t.Fatalf("malformed-list reconciliation did not verify every asset:\n%s", output) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if got := fake.uploadCalls[existingName]; got != 0 { + t.Errorf("upload calls for existing %s = %d, want 0", existingName, got) + } +} + +func TestReconcileGiteeAssetsDisablesExpectContinueForLargeUploads(t *testing.T) { + scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) + distDir := seedGiteeDist(t) + mustWriteFile( + t, + filepath.Join(distDir, requiredGiteeAssets[0]), + []byte(strings.Repeat("x", 2<<20)), + 0o644, + ) + fake := newFakeGiteeRelease(false, false) + fake.rejectExpectContinue = true + server := httptest.NewServer(fake) + defer server.Close() + + cmd := exec.Command("bash", scriptPath) + cmd.Env = giteeAssetEnv(distDir, server.URL, "1") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("large-asset reconciliation error = %v\noutput:\n%s", err, output) + } + if !strings.Contains(string(output), "all 8 verified") { + t.Fatalf("large-asset reconciliation did not verify every asset:\n%s", output) + } +} + func TestReconcileGiteeAssetsFailsWhenAnyUploadIsMissing(t *testing.T) { scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh")) distDir := seedGiteeDist(t) @@ -515,7 +687,53 @@ func TestGiteeReleaseWorkflowUsesImmutableTagsAndBoundedRetryBudget(t *testing.T } perAssetSeconds := shellDefaultInt(t, reconciler, "GITEE_ASSET_TIMEOUT_SECONDS") overallSeconds := shellDefaultInt(t, reconciler, "GITEE_OVERALL_TIMEOUT_SECONDS") - finalListSeconds := shellDefaultInt(t, reconciler, "GITEE_LIST_MAX_TIME") + listSeconds := shellDefaultInt(t, reconciler, "GITEE_LIST_MAX_TIME") + listRetries := shellDefaultInt(t, reconciler, "GITEE_LIST_RETRIES") + listRetryDelay := shellDefaultInt(t, reconciler, "GITEE_LIST_RETRY_DELAY") + listRetryWindow := shellDefaultInt(t, reconciler, "GITEE_LIST_RETRY_WINDOW_SECONDS") + uploadSeconds := shellDefaultInt(t, reconciler, "GITEE_UPLOAD_MAX_TIME") + verifySeconds := shellDefaultInt(t, reconciler, "GITEE_VERIFY_MAX_TIME") + postUploadVerifyAttempts := shellDefaultInt(t, reconciler, "GITEE_POST_UPLOAD_VERIFY_ATTEMPTS") + verifyRetryDelay := shellDefaultInt(t, reconciler, "GITEE_VERIFY_RETRY_DELAY") + uploadRetryDelay := shellDefaultInt(t, reconciler, "GITEE_UPLOAD_RETRY_DELAY") + const minimumLargeAssetUploadSeconds = 1200 + if uploadSeconds < minimumLargeAssetUploadSeconds { + t.Fatalf( + "Gitee upload deadline = %ds, want at least %ds for near-10 MiB release assets", + uploadSeconds, minimumLargeAssetUploadSeconds, + ) + } + const minimumTransientListOutageSeconds = 300 + if listRetryWindow < minimumTransientListOutageSeconds { + t.Fatalf( + "Gitee list recovery budget = %ds, want at least %ds for a transient API outage", + listRetryWindow, minimumTransientListOutageSeconds, + ) + } + if (listRetries-1)*listRetryDelay < listRetryWindow { + t.Fatalf( + "%d Gitee list attempts with %ds delay cannot span the configured %ds retry window after fast failures", + listRetries, listRetryDelay, listRetryWindow, + ) + } + completeListRecoveryBudget := listRetryWindow + oneSlowSuccessBudget := 2*completeListRecoveryBudget + uploadSeconds + verifySeconds + if oneSlowSuccessBudget > perAssetSeconds { + t.Fatalf( + "one complete slow upload budget = %ds, exceeds per-asset deadline %ds", + oneSlowSuccessBudget, perAssetSeconds, + ) + } + fastFailureRetryBudget := uploadSeconds + (postUploadVerifyAttempts+3)*listSeconds + + verifySeconds + (postUploadVerifyAttempts-1)*verifyRetryDelay + uploadRetryDelay + fastFailureRetryBudget += 2 * (completeListRecoveryBudget - listSeconds) + if fastFailureRetryBudget > perAssetSeconds { + t.Fatalf( + "fast-failure retry budget = %ds, exceeds per-asset deadline %ds", + fastFailureRetryBudget, perAssetSeconds, + ) + } + finalListSeconds := completeListRecoveryBudget completeAssetBudget := perAssetSeconds*len(requiredGiteeAssets) + finalListSeconds if completeAssetBudget > overallSeconds { t.Fatalf( @@ -530,15 +748,16 @@ func TestGiteeReleaseWorkflowUsesImmutableTagsAndBoundedRetryBudget(t *testing.T ) } - completeSyncBudget := tagSeconds + lookupSeconds + createSeconds + reconcileSeconds + childDeadlineReserveSeconds := shellDefaultInt(t, syncScript, "GITEE_CHILD_DEADLINE_RESERVE_SECONDS") + completeSyncBudget := tagSeconds + lookupSeconds + createSeconds + reconcileSeconds + childDeadlineReserveSeconds if completeSyncBudget > syncSeconds { t.Fatalf( - "complete sync budget = %ds (tag=%d + lookup=%d + create=%d + reconcile=%d), exceeds sync deadline %ds", - completeSyncBudget, tagSeconds, lookupSeconds, createSeconds, reconcileSeconds, syncSeconds, + "complete sync budget = %ds (tag=%d + lookup=%d + create=%d + reconcile=%d + child reserve=%d), exceeds sync deadline %ds", + completeSyncBudget, tagSeconds, lookupSeconds, createSeconds, reconcileSeconds, childDeadlineReserveSeconds, syncSeconds, ) } - const workflowReserveMinutes = 5 + const syncStepReserveMinutes = 4 releasePath := mustAbs(t, filepath.Join("..", "..", ".github", "workflows", "release.yml")) releaseData, err := os.ReadFile(releasePath) if err != nil { @@ -550,21 +769,49 @@ func TestGiteeReleaseWorkflowUsesImmutableTagsAndBoundedRetryBudget(t *testing.T "mirror-gitee-release:", []string{ "name: Check out sealed release source", + "name: Check out trusted release tooling", + "name: Fetch and verify sealed release tag", "name: Restore finalized distribution files", "name: Mirror release to Gitee (China)", }, - workflowReserveMinutes, + 17, ) releaseSyncStepSeconds := workflowTimeoutMinutesAfter( t, string(releaseData), "mirror-gitee-release:", "name: Mirror release to Gitee (China)", ) * 60 - if syncSeconds+workflowReserveMinutes*60 > releaseSyncStepSeconds { + if syncSeconds+syncStepReserveMinutes*60 > releaseSyncStepSeconds { t.Fatalf( "sync deadline %ds plus reserve exceeds release fallback step %ds", syncSeconds, releaseSyncStepSeconds, ) } + assertWorkflowBudget( + t, + string(releaseData), + "repair-channel:", + []string{ + "name: Check out trusted release tooling", + "name: Validate repair version", + "name: Verify immutable release authority", + "name: Check out sealed release source", + "name: Fetch and verify sealed release tag", + "name: Require successful Release workflow delivery", + "name: Download and verify immutable GitHub Release assets", + "name: Mirror release to Gitee (China)", + }, + 5, + ) + repairSyncStepSeconds := workflowTimeoutMinutesAfter( + t, string(releaseData), "repair-channel:", "name: Mirror release to Gitee (China)", + ) * 60 + if syncSeconds+syncStepReserveMinutes*60 > repairSyncStepSeconds { + t.Fatalf( + "sync deadline %ds plus reserve exceeds repair step %ds", + syncSeconds, repairSyncStepSeconds, + ) + } + localBuildPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "build-and-publish-gitee.sh")) localBuildData, err := os.ReadFile(localBuildPath) if err != nil { @@ -731,6 +978,9 @@ func giteeAssetEnv(distDir, apiURL, retries string) []string { "GITEE_RELEASE_ID=1", "GITEE_CURL_CONNECT_TIMEOUT=2", "GITEE_CURL_MAX_TIME=2", + "GITEE_LIST_RETRIES=2", + "GITEE_LIST_RETRY_DELAY=0", + "GITEE_LIST_RETRY_WINDOW_SECONDS=2", "GITEE_UPLOAD_MAX_TIME=2", "GITEE_UPLOAD_RETRIES="+retries, "GITEE_UPLOAD_RETRY_DELAY=0", @@ -747,14 +997,22 @@ type fakeGiteeAsset struct { } type fakeGiteeRelease struct { - mu sync.Mutex - nextID int - assets map[int]fakeGiteeAsset - uploadCalls map[string]int - dropFirstResponse bool - droppedResponse bool - failUploads bool - uploadDelay time.Duration + mu sync.Mutex + nextID int + assets map[int]fakeGiteeAsset + uploadCalls map[string]int + dropFirstResponse bool + droppedResponse bool + failUploads bool + uploadDelay time.Duration + rejectExpectContinue bool + listFailuresRemaining int + malformedListResponsesRemaining int + listErrorResponsesRemaining int + listEmptyResponsesRemaining int + listErrorResponsesAfterUpload int + listEmptyResponsesAfterUpload int + listCalls int } func newFakeGiteeRelease(dropFirstResponse, failUploads bool) *fakeGiteeRelease { @@ -799,7 +1057,42 @@ func (f *fakeGiteeRelease) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (f *fakeGiteeRelease) list(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() + f.listCalls++ + if f.listFailuresRemaining > 0 { + f.listFailuresRemaining-- + http.Error(w, "temporary Gitee list outage", http.StatusServiceUnavailable) + return + } + if f.listErrorResponsesRemaining > 0 { + f.listErrorResponsesRemaining-- + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{}`) + return + } + if f.listEmptyResponsesRemaining > 0 { + f.listEmptyResponsesRemaining-- + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]any{}) + return + } baseURL := requestBaseURL(r) + if f.malformedListResponsesRemaining > 0 { + f.malformedListResponsesRemaining-- + for _, asset := range f.assets { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]any{ + map[string]any{ + "id": asset.id, + "name": asset.name, + "browser_download_url": fmt.Sprintf("%s/download/%d", baseURL, asset.id), + }, + nil, + }) + return + } + http.Error(w, "malformed-list fixture requires one asset", http.StatusInternalServerError) + return + } rows := make([]map[string]any, 0, len(f.assets)) for _, asset := range f.assets { rows = append(rows, map[string]any{ @@ -813,6 +1106,10 @@ func (f *fakeGiteeRelease) list(w http.ResponseWriter, r *http.Request) { } func (f *fakeGiteeRelease) upload(w http.ResponseWriter, r *http.Request) { + if f.rejectExpectContinue && r.Header.Get("Expect") != "" { + http.Error(w, "Expect: 100-continue is not supported", http.StatusExpectationFailed) + return + } if err := r.ParseMultipartForm(32 << 20); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -847,6 +1144,14 @@ func (f *fakeGiteeRelease) upload(w http.ResponseWriter, r *http.Request) { id := f.nextID f.nextID++ f.assets[id] = fakeGiteeAsset{id: id, name: header.Filename, data: data} + if f.listErrorResponsesAfterUpload > 0 { + f.listErrorResponsesRemaining += f.listErrorResponsesAfterUpload + f.listErrorResponsesAfterUpload = 0 + } + if f.listEmptyResponsesAfterUpload > 0 { + f.listEmptyResponsesRemaining += f.listEmptyResponsesAfterUpload + f.listEmptyResponsesAfterUpload = 0 + } drop := f.dropFirstResponse && !f.droppedResponse if drop { f.droppedResponse = true