ci: shard full-suite coverage and cache merge-base profile
The Coverage context was the PR critical path (~17 min end to end): coverage-current re-ran the whole suite serially (-p 1, ~13 min) and coverage-baseline re-ran it again at the merge-base (~13 min) although that profile is a pure function of the base commit. - coverage-current now owns only the scoped (standard-tier) profile; full-suite candidate profiles come from a 5-way shard matrix (app/cli/generators/helpers/remaining) that keeps -p 1 inside each shard on isolated runners. scripts/ci/test-packages.sh list-coverage defines the shards and verify proves the union equals the previous single-run package set exactly once. - the aggregate Coverage job reassembles the disjoint shard profiles into coverage.txt before make coverage-gate, failing closed when a shard file is missing, so gate semantics (100% changed-code + scope-matched overall non-regression) are byte-compatible. - coverage-baseline restores the merge-base full-suite profile from an exact-key cache (merge-base SHA + resolved Go version) written by the last green main push; any miss falls back to recomputing in the merge-base worktree. Exact key only - no prefix fallback, a near-miss profile would compare the candidate against the wrong commit. - new contract tests pin the shard matrix, the assembly step, the exact-key cache pair, and the absence of restore-keys; the package plan test also covers the coverage shard partition.
This commit is contained in:
chichuan
@@ -14,6 +14,39 @@ func TestCITestPackagePlanCoversDefaultPackagesExactlyOnce(t *testing.T) {
|
||||
if !strings.Contains(output, "default packages exactly once") {
|
||||
t.Fatalf("verify output = %q, want coverage summary", output)
|
||||
}
|
||||
if !strings.Contains(output, "full-suite packages exactly once") {
|
||||
t.Fatalf("verify output = %q, want coverage shard plan summary", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCICoveragePackagePlanRoutesFullSuiteScope(t *testing.T) {
|
||||
root := testPackagePlanRoot(t)
|
||||
remaining := strings.Fields(runTestPackagePlan(t, root, "list-coverage", "remaining"))
|
||||
|
||||
for _, suffix := range []string{"/cmd", "/internal/output", "/skills"} {
|
||||
if !containsPackageSuffix(remaining, suffix) {
|
||||
t.Errorf("coverage remaining shard does not contain package ending in %q", suffix)
|
||||
}
|
||||
}
|
||||
for _, suffix := range []string{
|
||||
"/internal/app",
|
||||
"/internal/cli",
|
||||
"/internal/generator",
|
||||
"/internal/helpers",
|
||||
"/test/smoke",
|
||||
"/test/scripts",
|
||||
"/pkg/cmdutil",
|
||||
"/scripts/policy/coverage-gate",
|
||||
} {
|
||||
if containsPackageSuffix(remaining, suffix) {
|
||||
t.Errorf("coverage remaining shard unexpectedly contains package ending in %q", suffix)
|
||||
}
|
||||
}
|
||||
|
||||
app := strings.Fields(runTestPackagePlan(t, root, "list-coverage", "app"))
|
||||
if !containsPackageSuffix(app, "/internal/app") {
|
||||
t.Error("coverage app shard does not contain /internal/app")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCITestPackagePlanRoutesPublicTestSuites(t *testing.T) {
|
||||
|
||||
@@ -118,3 +118,97 @@ esac
|
||||
assertDiffProfiles(t, runGate(t, &empty), "coverage.txt")
|
||||
})
|
||||
}
|
||||
|
||||
// TestCoverageWorkflowShardsAndBaselineCache pins the full-suite coverage
|
||||
// architecture: the candidate profile is produced by disjoint per-shard
|
||||
// helper jobs and reassembled before enforcement, and the merge-base profile
|
||||
// is reused only through an exact-key cache written by a green main push of
|
||||
// that same commit. Near-miss reuse (restore-keys) would compare the
|
||||
// candidate against the wrong commit and must never appear.
|
||||
func TestCoverageWorkflowShardsAndBaselineCache(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatalf("Abs(repo root) error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "ci.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(ci.yml) error = %v", err)
|
||||
}
|
||||
admission := string(data)
|
||||
|
||||
currentStart := strings.Index(admission, "\n coverage-current:\n")
|
||||
fullStart := strings.Index(admission, "\n coverage-current-full:\n")
|
||||
supportingStart := strings.Index(admission, "\n coverage-supporting:\n")
|
||||
baselineStart := strings.Index(admission, "\n coverage-baseline:\n")
|
||||
gateStart := strings.Index(admission, "\n coverage:\n")
|
||||
policyStart := strings.Index(admission, "\n policy:\n")
|
||||
if currentStart < 0 || fullStart <= currentStart || supportingStart <= fullStart ||
|
||||
baselineStart <= supportingStart || gateStart <= baselineStart || policyStart <= gateStart {
|
||||
t.Fatal("CI workflow missing ordered coverage job boundaries")
|
||||
}
|
||||
|
||||
currentJob := admission[currentStart:fullStart]
|
||||
if !strings.Contains(currentJob, "needs.lint.outputs.full_suite != 'true'") {
|
||||
t.Error("coverage-current must be scoped-tier only; the full suite belongs to the shard matrix")
|
||||
}
|
||||
if strings.Contains(currentJob, "./ ./cmd/... ./internal/... ./skills/...") {
|
||||
t.Error("coverage-current must not retain the retired single serial full-suite run")
|
||||
}
|
||||
|
||||
fullJob := admission[fullStart:supportingStart]
|
||||
for _, want := range []string{
|
||||
"needs.lint.outputs.full_suite == 'true'",
|
||||
"fail-fast: false",
|
||||
" - app",
|
||||
" - cli",
|
||||
" - generators",
|
||||
" - helpers",
|
||||
" - remaining",
|
||||
`./scripts/ci/test-packages.sh list-coverage "$COVERAGE_SHARD"`,
|
||||
"go test -count=1 -p 1",
|
||||
`-coverprofile="coverage-shard-$COVERAGE_SHARD.txt"`,
|
||||
"-covermode=atomic",
|
||||
"name: coverage-current-shard-${{ matrix.shard }}",
|
||||
} {
|
||||
if !strings.Contains(fullJob, want) {
|
||||
t.Errorf("coverage-current-full missing shard contract %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
baselineJob := admission[baselineStart:gateStart]
|
||||
baselineKey := "dws-coverage-full-v1-${{ env.COVERAGE_BASE_REF }}-go${{ steps.setup-go.outputs.go-version }}"
|
||||
for _, want := range []string{
|
||||
"uses: actions/cache/restore@v4",
|
||||
"uses: actions/cache/save@v4",
|
||||
"path: coverage-base.txt",
|
||||
"key: " + baselineKey,
|
||||
"if: steps.baseline-cache.outputs.cache-hit != 'true'",
|
||||
} {
|
||||
if !strings.Contains(baselineJob, want) {
|
||||
t.Errorf("coverage-baseline missing cache contract %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Count(baselineJob, "key: "+baselineKey) != 2 {
|
||||
t.Error("coverage-baseline restore and save must use the identical exact cache key")
|
||||
}
|
||||
|
||||
gateJob := admission[gateStart:policyStart]
|
||||
for _, want := range []string{
|
||||
"pattern: coverage-current-*",
|
||||
"merge-multiple: true",
|
||||
"for shard in app cli generators helpers remaining; do",
|
||||
"test ! -f coverage.txt",
|
||||
`test "$(head -n 1 "$profile")" = "mode: atomic"`,
|
||||
"github.event_name == 'push'",
|
||||
"key: dws-coverage-full-v1-${{ github.sha }}-go${{ steps.setup-go.outputs.go-version }}",
|
||||
`"current shards:$CURRENT_FULL_RESULT:$current_full_expected"`,
|
||||
} {
|
||||
if !strings.Contains(gateJob, want) {
|
||||
t.Errorf("coverage gate missing shard assembly contract %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(admission, "restore-keys") {
|
||||
t.Error("coverage caches must stay exact-key; prefix restore-keys can resurrect a wrong-commit baseline")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user