ci: shard full-suite coverage and cache merge-base profile
The Coverage context was the PR critical path (~17 min end to end): coverage-current re-ran the whole suite serially (-p 1, ~13 min) and coverage-baseline re-ran it again at the merge-base (~13 min) although that profile is a pure function of the base commit. - coverage-current now owns only the scoped (standard-tier) profile; full-suite candidate profiles come from a 5-way shard matrix (app/cli/generators/helpers/remaining) that keeps -p 1 inside each shard on isolated runners. scripts/ci/test-packages.sh list-coverage defines the shards and verify proves the union equals the previous single-run package set exactly once. - the aggregate Coverage job reassembles the disjoint shard profiles into coverage.txt before make coverage-gate, failing closed when a shard file is missing, so gate semantics (100% changed-code + scope-matched overall non-regression) are byte-compatible. - coverage-baseline restores the merge-base full-suite profile from an exact-key cache (merge-base SHA + resolved Go version) written by the last green main push; any miss falls back to recomputing in the merge-base worktree. Exact key only - no prefix fallback, a near-miss profile would compare the candidate against the wrong commit. - new contract tests pin the shard matrix, the assembly step, the exact-key cache pair, and the absence of restore-keys; the package plan test also covers the coverage shard partition.
This commit is contained in:
chichuan
+147
-38
@@ -911,7 +911,7 @@ jobs:
|
||||
coverage-current:
|
||||
name: Coverage (current)
|
||||
needs: lint
|
||||
if: ${{ needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true' }}
|
||||
if: ${{ needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true' && needs.lint.outputs.full_suite != 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
@@ -926,10 +926,6 @@ jobs:
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Install archive tooling
|
||||
if: needs.lint.outputs.full_suite == 'true'
|
||||
run: sudo apt-get update && sudo apt-get install -y zip unzip
|
||||
|
||||
- name: Resolve authoritative coverage base
|
||||
env:
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
@@ -951,41 +947,33 @@ jobs:
|
||||
- name: Build
|
||||
run: make build
|
||||
|
||||
- name: Run current unit tests with coverage
|
||||
- name: Run scoped unit tests with coverage
|
||||
shell: bash
|
||||
env:
|
||||
DWS_PACKAGE_VERSION: 0.0.0-test
|
||||
FULL_SUITE: ${{ needs.lint.outputs.full_suite }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$FULL_SUITE" = true ]; then
|
||||
changed_output="$(
|
||||
./scripts/ci/changed-test-packages.sh \
|
||||
changed "$COVERAGE_BASE_REF" "$COVERAGE_HEAD_REF"
|
||||
)"
|
||||
impacted_output="$(
|
||||
./scripts/ci/changed-test-packages.sh \
|
||||
list "$COVERAGE_BASE_REF" "$COVERAGE_HEAD_REF"
|
||||
)"
|
||||
if [ -z "$changed_output" ] || [ -z "$impacted_output" ]; then
|
||||
printf 'mode: atomic\n' > coverage.txt
|
||||
echo "No buildable Go package needs scoped coverage." \
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
mapfile -t changed_packages <<< "$changed_output"
|
||||
mapfile -t impacted_packages <<< "$impacted_output"
|
||||
coverpkg="$(IFS=,; echo "${changed_packages[*]}")"
|
||||
go test -count=1 -p 1 \
|
||||
-coverpkg="$coverpkg" \
|
||||
-coverprofile=coverage.txt \
|
||||
-covermode=atomic \
|
||||
./ ./cmd/... ./internal/... ./skills/...
|
||||
else
|
||||
changed_output="$(
|
||||
./scripts/ci/changed-test-packages.sh \
|
||||
changed "$COVERAGE_BASE_REF" "$COVERAGE_HEAD_REF"
|
||||
)"
|
||||
impacted_output="$(
|
||||
./scripts/ci/changed-test-packages.sh \
|
||||
list "$COVERAGE_BASE_REF" "$COVERAGE_HEAD_REF"
|
||||
)"
|
||||
if [ -z "$changed_output" ] || [ -z "$impacted_output" ]; then
|
||||
printf 'mode: atomic\n' > coverage.txt
|
||||
echo "No buildable Go package needs scoped coverage." \
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
mapfile -t changed_packages <<< "$changed_output"
|
||||
mapfile -t impacted_packages <<< "$impacted_output"
|
||||
coverpkg="$(IFS=,; echo "${changed_packages[*]}")"
|
||||
go test -count=1 -p 1 \
|
||||
-coverpkg="$coverpkg" \
|
||||
-coverprofile=coverage.txt \
|
||||
-covermode=atomic \
|
||||
"${impacted_packages[@]}"
|
||||
fi
|
||||
"${impacted_packages[@]}"
|
||||
fi
|
||||
if [ "$(wc -l < coverage.txt)" -gt 1 ]; then
|
||||
go tool cover -func=coverage.txt
|
||||
@@ -998,6 +986,66 @@ jobs:
|
||||
path: coverage.txt
|
||||
retention-days: 1
|
||||
|
||||
coverage-current-full:
|
||||
name: "Coverage (current: ${{ matrix.shard }})"
|
||||
needs: lint
|
||||
if: ${{ needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true' && needs.lint.outputs.full_suite == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard:
|
||||
- app
|
||||
- cli
|
||||
- generators
|
||||
- helpers
|
||||
- remaining
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Install archive tooling
|
||||
run: sudo apt-get update && sudo apt-get install -y zip unzip
|
||||
|
||||
- name: Build
|
||||
run: make build
|
||||
|
||||
# Each shard keeps -p 1 so the authoritative measurement stays serial
|
||||
# inside one instrumented process group; shards run on isolated runners,
|
||||
# and scripts/ci/test-packages.sh verify proves the shard union equals
|
||||
# the previous single full-suite package set exactly once.
|
||||
- name: Run current shard tests with coverage
|
||||
shell: bash
|
||||
env:
|
||||
DWS_PACKAGE_VERSION: 0.0.0-test
|
||||
COVERAGE_SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
package_output="$(./scripts/ci/test-packages.sh list-coverage "$COVERAGE_SHARD")"
|
||||
test -n "$package_output"
|
||||
mapfile -t packages <<< "$package_output"
|
||||
test "${#packages[@]}" -gt 0
|
||||
go test -count=1 -p 1 \
|
||||
-coverprofile="coverage-shard-$COVERAGE_SHARD.txt" \
|
||||
-covermode=atomic \
|
||||
"${packages[@]}"
|
||||
go tool cover -func="coverage-shard-$COVERAGE_SHARD.txt" | tail -n 1
|
||||
|
||||
- name: Upload current shard coverage profile
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-current-shard-${{ matrix.shard }}
|
||||
path: coverage-shard-${{ matrix.shard }}.txt
|
||||
retention-days: 1
|
||||
|
||||
coverage-supporting:
|
||||
name: Coverage (supporting)
|
||||
needs: lint
|
||||
@@ -1051,14 +1099,11 @@ jobs:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Set up Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Install archive tooling
|
||||
if: needs.lint.outputs.full_suite == 'true'
|
||||
run: sudo apt-get update && sudo apt-get install -y zip unzip
|
||||
|
||||
- name: Resolve authoritative coverage base
|
||||
env:
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
@@ -1076,7 +1121,26 @@ jobs:
|
||||
git rev-parse --verify "${base_ref}^{commit}" >/dev/null
|
||||
echo "COVERAGE_BASE_REF=$base_ref" >> "$GITHUB_ENV"
|
||||
|
||||
# The merge-base full-suite profile is a pure function of the base
|
||||
# commit. Reuse the profile published by the last green push run of
|
||||
# exactly that commit instead of re-running the whole suite; any key
|
||||
# mismatch falls back to authoritative recomputation. Exact key only,
|
||||
# never prefix fallback: a near-miss profile would compare the
|
||||
# candidate against the wrong commit.
|
||||
- name: Restore cached merge-base coverage profile
|
||||
id: baseline-cache
|
||||
if: needs.lint.outputs.full_suite == 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: coverage-base.txt
|
||||
key: dws-coverage-full-v1-${{ env.COVERAGE_BASE_REF }}-go${{ steps.setup-go.outputs.go-version }}
|
||||
|
||||
- name: Install archive tooling
|
||||
if: needs.lint.outputs.full_suite == 'true' && steps.baseline-cache.outputs.cache-hit != 'true'
|
||||
run: sudo apt-get update && sudo apt-get install -y zip unzip
|
||||
|
||||
- name: Run baseline unit tests with coverage
|
||||
if: steps.baseline-cache.outputs.cache-hit != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
DWS_PACKAGE_VERSION: 0.0.0-test
|
||||
@@ -1125,6 +1189,13 @@ jobs:
|
||||
fi
|
||||
)
|
||||
|
||||
- name: Save merge-base coverage profile cache
|
||||
if: needs.lint.outputs.full_suite == 'true' && steps.baseline-cache.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: coverage-base.txt
|
||||
key: dws-coverage-full-v1-${{ env.COVERAGE_BASE_REF }}-go${{ steps.setup-go.outputs.go-version }}
|
||||
|
||||
- name: Upload baseline coverage profile
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
@@ -1137,6 +1208,7 @@ jobs:
|
||||
needs:
|
||||
- lint
|
||||
- coverage-current
|
||||
- coverage-current-full
|
||||
- coverage-supporting
|
||||
- coverage-baseline
|
||||
- coverage-darwin
|
||||
@@ -1152,6 +1224,7 @@ jobs:
|
||||
FULL_SUITE: ${{ needs.lint.outputs.full_suite }}
|
||||
PLATFORM_SENSITIVE: ${{ needs.lint.outputs.platform_sensitive }}
|
||||
CURRENT_RESULT: ${{ needs.coverage-current.result }}
|
||||
CURRENT_FULL_RESULT: ${{ needs.coverage-current-full.result }}
|
||||
SUPPORTING_RESULT: ${{ needs.coverage-supporting.result }}
|
||||
BASELINE_RESULT: ${{ needs.coverage-baseline.result }}
|
||||
DARWIN_RESULT: ${{ needs.coverage-darwin.result }}
|
||||
@@ -1159,6 +1232,7 @@ jobs:
|
||||
run: |
|
||||
failed=0
|
||||
current_expected=success
|
||||
current_full_expected=skipped
|
||||
supporting_expected=skipped
|
||||
baseline_expected=success
|
||||
native_expected=skipped
|
||||
@@ -1166,6 +1240,8 @@ jobs:
|
||||
current_expected=skipped
|
||||
baseline_expected=skipped
|
||||
elif [ "$FULL_SUITE" = true ]; then
|
||||
current_expected=skipped
|
||||
current_full_expected=success
|
||||
supporting_expected=success
|
||||
fi
|
||||
if [ "$CHANGELOG_ONLY" != true ] &&
|
||||
@@ -1176,6 +1252,7 @@ jobs:
|
||||
|
||||
for profile in \
|
||||
"current:$CURRENT_RESULT:$current_expected" \
|
||||
"current shards:$CURRENT_FULL_RESULT:$current_full_expected" \
|
||||
"supporting:$SUPPORTING_RESULT:$supporting_expected" \
|
||||
"baseline:$BASELINE_RESULT:$baseline_expected"
|
||||
do
|
||||
@@ -1211,6 +1288,7 @@ jobs:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Set up Go
|
||||
id: setup-go
|
||||
if: needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true'
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
@@ -1234,11 +1312,12 @@ jobs:
|
||||
git rev-parse --verify "${base_ref}^{commit}" >/dev/null
|
||||
echo "COVERAGE_BASE_REF=$base_ref" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download current coverage profile
|
||||
- name: Download current coverage profiles
|
||||
if: needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: coverage-current-profile
|
||||
pattern: coverage-current-*
|
||||
merge-multiple: true
|
||||
path: .
|
||||
|
||||
- name: Download supporting coverage profiles
|
||||
@@ -1255,6 +1334,26 @@ jobs:
|
||||
name: coverage-baseline-profile
|
||||
path: .
|
||||
|
||||
# Shard profiles cover disjoint package sets, so their block-level
|
||||
# concatenation is the same candidate profile one serial run produced.
|
||||
# Every expected shard must be present; a missing shard would silently
|
||||
# shrink the scope-matched overall comparison.
|
||||
- name: Assemble full-suite coverage profile
|
||||
if: needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true' && needs.lint.outputs.full_suite == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test ! -f coverage.txt
|
||||
for shard in app cli generators helpers remaining; do
|
||||
profile="coverage-shard-$shard.txt"
|
||||
test -f "$profile"
|
||||
test "$(head -n 1 "$profile")" = "mode: atomic"
|
||||
done
|
||||
printf 'mode: atomic\n' > coverage.txt
|
||||
for shard in app cli generators helpers remaining; do
|
||||
tail -n +2 "coverage-shard-$shard.txt" >> coverage.txt
|
||||
done
|
||||
|
||||
- name: Enforce coverage gate
|
||||
if: needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true'
|
||||
env:
|
||||
@@ -1275,6 +1374,16 @@ jobs:
|
||||
COVERAGE_ADDITIONAL_DIFF_PROFILE="$additional_profile" \
|
||||
make coverage-gate BASE_REF="$COVERAGE_BASE_REF"
|
||||
|
||||
# Publish this push's full-suite profile as the merge-base cache for
|
||||
# future PRs whose merge-base is exactly this commit. Saved only after
|
||||
# the gate passed so a broken run never becomes a baseline.
|
||||
- name: Save push coverage profile as merge-base cache
|
||||
if: github.event_name == 'push' && needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true' && needs.lint.outputs.full_suite == 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: coverage.txt
|
||||
key: dws-coverage-full-v1-${{ github.sha }}-go${{ steps.setup-go.outputs.go-version }}
|
||||
|
||||
- name: Generate coverage report
|
||||
if: needs.lint.outputs.changelog_only != 'true' && needs.lint.outputs.docs_only != 'true'
|
||||
run: |
|
||||
|
||||
@@ -18,7 +18,7 @@ help:
|
||||
@printf "Available targets:\n"
|
||||
@printf " make build - Build the dws CLI binary\n"
|
||||
@printf " make test - Run the Go test suite\n"
|
||||
@printf " make test-plan - Verify every default Go package belongs to one CI test shard\n"
|
||||
@printf " make test-plan - Verify CI test and full-suite coverage package plans cover their scopes exactly once\n"
|
||||
@printf " make test-auth-legacy-compat - Run stable legacy authentication compatibility regressions\n"
|
||||
@printf " make lint - Run formatting checks, go vet, and staticcheck\n"
|
||||
@printf " make format-check - Check all repository Go source files with gofmt\n"
|
||||
|
||||
+12
-2
@@ -201,8 +201,18 @@ base_ref=$(git merge-base HEAD origin/main)
|
||||
standard PR, CI derives changed packages and their reverse-dependency test
|
||||
closure, then generates candidate and merge-base profiles with the same test
|
||||
scope and `coverpkg`. High-risk and protected-main runs use the complete
|
||||
profiles. Supporting and (when platform-selected) native profiles are
|
||||
generated before the aggregate `Coverage` context evaluates them. The
|
||||
profiles. The complete candidate profile is produced by disjoint per-shard
|
||||
helper jobs (`scripts/ci/test-packages.sh list-coverage`, kept serial with
|
||||
`-p 1` inside each shard; `verify` proves the shard union equals the
|
||||
full-suite scope exactly once) and concatenated in the aggregate job before
|
||||
enforcement. The complete merge-base profile is restored from an exact-key
|
||||
cache written by the last green `main` push of that same commit (key:
|
||||
merge-base SHA plus resolved Go version); any miss falls back to recomputing
|
||||
it in a merge-base worktree. Near-miss reuse is forbidden — the caches carry
|
||||
no prefix restore keys, because a neighbouring commit's profile would compare
|
||||
the candidate against the wrong baseline. Supporting and (when
|
||||
platform-selected) native profiles are generated before the aggregate
|
||||
`Coverage` context evaluates them. The
|
||||
aggregate and native gates require 100% coverage for changed executable Go
|
||||
statements. Overall coverage remains an unrounded, zero-tolerance,
|
||||
scope-matched merge-base non-regression check. Candidate and baseline profiles
|
||||
|
||||
@@ -7,6 +7,7 @@ MODULE="$(cd "$ROOT" && go list -m)"
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
"usage: $0 list <app|generators|helpers|cli|smoke|remaining|release-scripts>" \
|
||||
" $0 list-coverage <app|cli|generators|helpers|remaining>" \
|
||||
" $0 verify" >&2
|
||||
exit 2
|
||||
}
|
||||
@@ -57,6 +58,52 @@ list_shard() {
|
||||
esac
|
||||
}
|
||||
|
||||
# Full-suite coverage measurement shards the same authoritative package set the
|
||||
# single serial run used: ./ ./cmd/... ./internal/... ./skills/... (pkg/ and
|
||||
# scripts/ belong to the supporting profile; test/ suites carry no production
|
||||
# statements for the candidate profile). Shards stay disjoint so their merged
|
||||
# profile is block-for-block equivalent to one serial invocation.
|
||||
list_coverage_scope() {
|
||||
cd "$ROOT"
|
||||
go list ./ ./cmd/... ./internal/... ./skills/...
|
||||
}
|
||||
|
||||
list_coverage_shard() {
|
||||
shard="$1"
|
||||
cd "$ROOT"
|
||||
|
||||
case "$shard" in
|
||||
app)
|
||||
go list ./internal/app/...
|
||||
;;
|
||||
cli)
|
||||
go list ./internal/cli/...
|
||||
;;
|
||||
generators)
|
||||
go list ./internal/generator/...
|
||||
;;
|
||||
helpers)
|
||||
go list ./internal/helpers/...
|
||||
;;
|
||||
remaining)
|
||||
scope_packages="$(list_coverage_scope)"
|
||||
printf '%s\n' "$scope_packages" | while IFS= read -r package; do
|
||||
case "$package" in
|
||||
"$MODULE/internal/app"|"$MODULE/internal/app/"*) ;;
|
||||
"$MODULE/internal/cli"|"$MODULE/internal/cli/"*) ;;
|
||||
"$MODULE/internal/generator"|"$MODULE/internal/generator/"*) ;;
|
||||
"$MODULE/internal/helpers"|"$MODULE/internal/helpers/"*) ;;
|
||||
*) printf '%s\n' "$package" ;;
|
||||
esac
|
||||
done
|
||||
;;
|
||||
*)
|
||||
printf 'unknown coverage package shard: %s\n' "$shard" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
verify_plan() {
|
||||
workdir="$(mktemp -d "${TMPDIR:-/tmp}/dws-test-packages.XXXXXX")"
|
||||
trap 'rm -rf "$workdir"' EXIT HUP INT TERM
|
||||
@@ -114,6 +161,58 @@ verify_plan() {
|
||||
|
||||
package_count="$(wc -l < "$expected" | tr -d ' ')"
|
||||
printf 'test package plan covers %s default packages exactly once\n' "$package_count"
|
||||
|
||||
coverage_expected="$workdir/coverage-expected"
|
||||
coverage_assigned="$workdir/coverage-assigned"
|
||||
coverage_unique="$workdir/coverage-unique"
|
||||
coverage_duplicates="$workdir/coverage-duplicates"
|
||||
coverage_missing="$workdir/coverage-missing"
|
||||
coverage_unexpected="$workdir/coverage-unexpected"
|
||||
|
||||
list_coverage_scope > "$workdir/coverage-scope"
|
||||
LC_ALL=C sort -u "$workdir/coverage-scope" > "$coverage_expected"
|
||||
: > "$coverage_assigned"
|
||||
|
||||
for shard in app cli generators helpers remaining; do
|
||||
shard_packages="$workdir/coverage-$shard"
|
||||
unsorted_shard_packages="$workdir/coverage-$shard.unsorted"
|
||||
list_coverage_shard "$shard" > "$unsorted_shard_packages"
|
||||
LC_ALL=C sort "$unsorted_shard_packages" > "$shard_packages"
|
||||
if [ ! -s "$shard_packages" ]; then
|
||||
printf 'coverage package shard is empty: %s\n' "$shard" >&2
|
||||
exit 1
|
||||
fi
|
||||
cat "$shard_packages" >> "$coverage_assigned"
|
||||
done
|
||||
|
||||
LC_ALL=C sort "$coverage_assigned" -o "$coverage_assigned"
|
||||
uniq -d "$coverage_assigned" > "$coverage_duplicates"
|
||||
LC_ALL=C sort -u "$coverage_assigned" > "$coverage_unique"
|
||||
comm -23 "$coverage_expected" "$coverage_unique" > "$coverage_missing"
|
||||
comm -13 "$coverage_expected" "$coverage_unique" > "$coverage_unexpected"
|
||||
|
||||
coverage_failed=0
|
||||
if [ -s "$coverage_duplicates" ]; then
|
||||
printf '%s\n' 'coverage packages assigned to more than one shard:' >&2
|
||||
sed 's/^/ /' "$coverage_duplicates" >&2
|
||||
coverage_failed=1
|
||||
fi
|
||||
if [ -s "$coverage_missing" ]; then
|
||||
printf '%s\n' 'full-suite coverage packages missing from the CI coverage plan:' >&2
|
||||
sed 's/^/ /' "$coverage_missing" >&2
|
||||
coverage_failed=1
|
||||
fi
|
||||
if [ -s "$coverage_unexpected" ]; then
|
||||
printf '%s\n' 'CI coverage plan contains packages outside its scope:' >&2
|
||||
sed 's/^/ /' "$coverage_unexpected" >&2
|
||||
coverage_failed=1
|
||||
fi
|
||||
if [ "$coverage_failed" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
coverage_count="$(wc -l < "$coverage_expected" | tr -d ' ')"
|
||||
printf 'coverage package plan covers %s full-suite packages exactly once\n' "$coverage_count"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
@@ -121,6 +220,10 @@ case "${1:-}" in
|
||||
[ "$#" -eq 2 ] || usage
|
||||
list_shard "$2"
|
||||
;;
|
||||
list-coverage)
|
||||
[ "$#" -eq 2 ] || usage
|
||||
list_coverage_shard "$2"
|
||||
;;
|
||||
verify)
|
||||
[ "$#" -eq 1 ] || usage
|
||||
verify_plan
|
||||
|
||||
@@ -14,6 +14,39 @@ func TestCITestPackagePlanCoversDefaultPackagesExactlyOnce(t *testing.T) {
|
||||
if !strings.Contains(output, "default packages exactly once") {
|
||||
t.Fatalf("verify output = %q, want coverage summary", output)
|
||||
}
|
||||
if !strings.Contains(output, "full-suite packages exactly once") {
|
||||
t.Fatalf("verify output = %q, want coverage shard plan summary", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCICoveragePackagePlanRoutesFullSuiteScope(t *testing.T) {
|
||||
root := testPackagePlanRoot(t)
|
||||
remaining := strings.Fields(runTestPackagePlan(t, root, "list-coverage", "remaining"))
|
||||
|
||||
for _, suffix := range []string{"/cmd", "/internal/output", "/skills"} {
|
||||
if !containsPackageSuffix(remaining, suffix) {
|
||||
t.Errorf("coverage remaining shard does not contain package ending in %q", suffix)
|
||||
}
|
||||
}
|
||||
for _, suffix := range []string{
|
||||
"/internal/app",
|
||||
"/internal/cli",
|
||||
"/internal/generator",
|
||||
"/internal/helpers",
|
||||
"/test/smoke",
|
||||
"/test/scripts",
|
||||
"/pkg/cmdutil",
|
||||
"/scripts/policy/coverage-gate",
|
||||
} {
|
||||
if containsPackageSuffix(remaining, suffix) {
|
||||
t.Errorf("coverage remaining shard unexpectedly contains package ending in %q", suffix)
|
||||
}
|
||||
}
|
||||
|
||||
app := strings.Fields(runTestPackagePlan(t, root, "list-coverage", "app"))
|
||||
if !containsPackageSuffix(app, "/internal/app") {
|
||||
t.Error("coverage app shard does not contain /internal/app")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCITestPackagePlanRoutesPublicTestSuites(t *testing.T) {
|
||||
|
||||
@@ -118,3 +118,97 @@ esac
|
||||
assertDiffProfiles(t, runGate(t, &empty), "coverage.txt")
|
||||
})
|
||||
}
|
||||
|
||||
// TestCoverageWorkflowShardsAndBaselineCache pins the full-suite coverage
|
||||
// architecture: the candidate profile is produced by disjoint per-shard
|
||||
// helper jobs and reassembled before enforcement, and the merge-base profile
|
||||
// is reused only through an exact-key cache written by a green main push of
|
||||
// that same commit. Near-miss reuse (restore-keys) would compare the
|
||||
// candidate against the wrong commit and must never appear.
|
||||
func TestCoverageWorkflowShardsAndBaselineCache(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatalf("Abs(repo root) error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "ci.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(ci.yml) error = %v", err)
|
||||
}
|
||||
admission := string(data)
|
||||
|
||||
currentStart := strings.Index(admission, "\n coverage-current:\n")
|
||||
fullStart := strings.Index(admission, "\n coverage-current-full:\n")
|
||||
supportingStart := strings.Index(admission, "\n coverage-supporting:\n")
|
||||
baselineStart := strings.Index(admission, "\n coverage-baseline:\n")
|
||||
gateStart := strings.Index(admission, "\n coverage:\n")
|
||||
policyStart := strings.Index(admission, "\n policy:\n")
|
||||
if currentStart < 0 || fullStart <= currentStart || supportingStart <= fullStart ||
|
||||
baselineStart <= supportingStart || gateStart <= baselineStart || policyStart <= gateStart {
|
||||
t.Fatal("CI workflow missing ordered coverage job boundaries")
|
||||
}
|
||||
|
||||
currentJob := admission[currentStart:fullStart]
|
||||
if !strings.Contains(currentJob, "needs.lint.outputs.full_suite != 'true'") {
|
||||
t.Error("coverage-current must be scoped-tier only; the full suite belongs to the shard matrix")
|
||||
}
|
||||
if strings.Contains(currentJob, "./ ./cmd/... ./internal/... ./skills/...") {
|
||||
t.Error("coverage-current must not retain the retired single serial full-suite run")
|
||||
}
|
||||
|
||||
fullJob := admission[fullStart:supportingStart]
|
||||
for _, want := range []string{
|
||||
"needs.lint.outputs.full_suite == 'true'",
|
||||
"fail-fast: false",
|
||||
" - app",
|
||||
" - cli",
|
||||
" - generators",
|
||||
" - helpers",
|
||||
" - remaining",
|
||||
`./scripts/ci/test-packages.sh list-coverage "$COVERAGE_SHARD"`,
|
||||
"go test -count=1 -p 1",
|
||||
`-coverprofile="coverage-shard-$COVERAGE_SHARD.txt"`,
|
||||
"-covermode=atomic",
|
||||
"name: coverage-current-shard-${{ matrix.shard }}",
|
||||
} {
|
||||
if !strings.Contains(fullJob, want) {
|
||||
t.Errorf("coverage-current-full missing shard contract %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
baselineJob := admission[baselineStart:gateStart]
|
||||
baselineKey := "dws-coverage-full-v1-${{ env.COVERAGE_BASE_REF }}-go${{ steps.setup-go.outputs.go-version }}"
|
||||
for _, want := range []string{
|
||||
"uses: actions/cache/restore@v4",
|
||||
"uses: actions/cache/save@v4",
|
||||
"path: coverage-base.txt",
|
||||
"key: " + baselineKey,
|
||||
"if: steps.baseline-cache.outputs.cache-hit != 'true'",
|
||||
} {
|
||||
if !strings.Contains(baselineJob, want) {
|
||||
t.Errorf("coverage-baseline missing cache contract %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Count(baselineJob, "key: "+baselineKey) != 2 {
|
||||
t.Error("coverage-baseline restore and save must use the identical exact cache key")
|
||||
}
|
||||
|
||||
gateJob := admission[gateStart:policyStart]
|
||||
for _, want := range []string{
|
||||
"pattern: coverage-current-*",
|
||||
"merge-multiple: true",
|
||||
"for shard in app cli generators helpers remaining; do",
|
||||
"test ! -f coverage.txt",
|
||||
`test "$(head -n 1 "$profile")" = "mode: atomic"`,
|
||||
"github.event_name == 'push'",
|
||||
"key: dws-coverage-full-v1-${{ github.sha }}-go${{ steps.setup-go.outputs.go-version }}",
|
||||
`"current shards:$CURRENT_FULL_RESULT:$current_full_expected"`,
|
||||
} {
|
||||
if !strings.Contains(gateJob, want) {
|
||||
t.Errorf("coverage gate missing shard assembly contract %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(admission, "restore-keys") {
|
||||
t.Error("coverage caches must stay exact-key; prefix restore-keys can resurrect a wrong-commit baseline")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user