The Coverage context was the PR critical path (~17 min end to end): coverage-current re-ran the whole suite serially (-p 1, ~13 min) and coverage-baseline re-ran it again at the merge-base (~13 min) although that profile is a pure function of the base commit. - coverage-current now owns only the scoped (standard-tier) profile; full-suite candidate profiles come from a 5-way shard matrix (app/cli/generators/helpers/remaining) that keeps -p 1 inside each shard on isolated runners. scripts/ci/test-packages.sh list-coverage defines the shards and verify proves the union equals the previous single-run package set exactly once. - the aggregate Coverage job reassembles the disjoint shard profiles into coverage.txt before make coverage-gate, failing closed when a shard file is missing, so gate semantics (100% changed-code + scope-matched overall non-regression) are byte-compatible. - coverage-baseline restores the merge-base full-suite profile from an exact-key cache (merge-base SHA + resolved Go version) written by the last green main push; any miss falls back to recomputing in the merge-base worktree. Exact key only - no prefix fallback, a near-miss profile would compare the candidate against the wrong commit. - new contract tests pin the shard matrix, the assembly step, the exact-key cache pair, and the absence of restore-keys; the package plan test also covers the coverage shard partition.
165 lines
5.1 KiB
Go
165 lines
5.1 KiB
Go
package scripts_test
|
|
|
|
import (
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestCITestPackagePlanCoversDefaultPackagesExactlyOnce(t *testing.T) {
|
|
root := testPackagePlanRoot(t)
|
|
output := runTestPackagePlan(t, root, "verify")
|
|
if !strings.Contains(output, "default packages exactly once") {
|
|
t.Fatalf("verify output = %q, want coverage summary", output)
|
|
}
|
|
if !strings.Contains(output, "full-suite packages exactly once") {
|
|
t.Fatalf("verify output = %q, want coverage shard plan summary", output)
|
|
}
|
|
}
|
|
|
|
func TestCICoveragePackagePlanRoutesFullSuiteScope(t *testing.T) {
|
|
root := testPackagePlanRoot(t)
|
|
remaining := strings.Fields(runTestPackagePlan(t, root, "list-coverage", "remaining"))
|
|
|
|
for _, suffix := range []string{"/cmd", "/internal/output", "/skills"} {
|
|
if !containsPackageSuffix(remaining, suffix) {
|
|
t.Errorf("coverage remaining shard does not contain package ending in %q", suffix)
|
|
}
|
|
}
|
|
for _, suffix := range []string{
|
|
"/internal/app",
|
|
"/internal/cli",
|
|
"/internal/generator",
|
|
"/internal/helpers",
|
|
"/test/smoke",
|
|
"/test/scripts",
|
|
"/pkg/cmdutil",
|
|
"/scripts/policy/coverage-gate",
|
|
} {
|
|
if containsPackageSuffix(remaining, suffix) {
|
|
t.Errorf("coverage remaining shard unexpectedly contains package ending in %q", suffix)
|
|
}
|
|
}
|
|
|
|
app := strings.Fields(runTestPackagePlan(t, root, "list-coverage", "app"))
|
|
if !containsPackageSuffix(app, "/internal/app") {
|
|
t.Error("coverage app shard does not contain /internal/app")
|
|
}
|
|
}
|
|
|
|
func TestCITestPackagePlanRoutesPublicTestSuites(t *testing.T) {
|
|
root := testPackagePlanRoot(t)
|
|
remaining := strings.Fields(runTestPackagePlan(t, root, "list", "remaining"))
|
|
smoke := strings.Fields(runTestPackagePlan(t, root, "list", "smoke"))
|
|
releaseScripts := strings.Fields(runTestPackagePlan(t, root, "list", "release-scripts"))
|
|
|
|
for _, suffix := range []string{
|
|
"/test/cli",
|
|
"/test/contract",
|
|
"/test/integration/extensions",
|
|
"/test/mock_mcp",
|
|
"/test/unit",
|
|
} {
|
|
if !containsPackageSuffix(remaining, suffix) {
|
|
t.Errorf("remaining shard does not contain package ending in %q", suffix)
|
|
}
|
|
}
|
|
if containsPackageSuffix(remaining, "/test/smoke") {
|
|
t.Error("remaining shard unexpectedly contains /test/smoke")
|
|
}
|
|
if containsPackageSuffix(remaining, "/test/scripts") {
|
|
t.Error("remaining shard unexpectedly contains /test/scripts")
|
|
}
|
|
if !containsPackageSuffix(smoke, "/test/smoke") {
|
|
t.Error("smoke shard does not contain /test/smoke")
|
|
}
|
|
if !containsPackageSuffix(releaseScripts, "/test/scripts") {
|
|
t.Error("release-scripts shard does not contain /test/scripts")
|
|
}
|
|
}
|
|
|
|
func TestCIAppRacePartitionsCoverTopLevelTestsExactlyOnce(t *testing.T) {
|
|
root := testPackagePlanRoot(t)
|
|
packages := strings.Fields(runTestPackagePlan(t, root, "list", "app"))
|
|
if len(packages) != 1 {
|
|
t.Fatalf("app package shard = %v, want exactly one package", packages)
|
|
}
|
|
|
|
script := filepath.Join(root, "scripts", "ci", "run-app-race-tests.sh")
|
|
cmd := exec.Command("sh", script, "verify", packages[0])
|
|
cmd.Dir = root
|
|
output, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
t.Fatalf("%s verify %s failed: %v\n%s", script, packages[0], err, output)
|
|
}
|
|
if !strings.Contains(string(output), "top-level tests exactly once") {
|
|
t.Fatalf("verify output = %q, want exact coverage summary", output)
|
|
}
|
|
}
|
|
|
|
func TestCITestPackagePlanFailsClosedWhenGoListFails(t *testing.T) {
|
|
root := testPackagePlanRoot(t)
|
|
fakeBin := t.TempDir()
|
|
fakeGo := filepath.Join(fakeBin, "go")
|
|
err := os.WriteFile(fakeGo, []byte(`#!/bin/sh
|
|
if [ "$1" = "list" ] && [ "$2" = "-m" ]; then
|
|
printf '%s\n' 'github.com/DingTalk-Real-AI/dingtalk-workspace-cli'
|
|
exit 0
|
|
fi
|
|
printf '%s\n' 'injected go list failure' >&2
|
|
exit 42
|
|
`), 0o755)
|
|
if err != nil {
|
|
t.Fatalf("write fake go: %v", err)
|
|
}
|
|
|
|
script := filepath.Join(root, "scripts", "ci", "test-packages.sh")
|
|
for _, args := range [][]string{{"list", "remaining"}, {"verify"}} {
|
|
cmd := exec.Command("sh", append([]string{script}, args...)...)
|
|
cmd.Dir = root
|
|
cmd.Env = []string{
|
|
"PATH=" + fakeBin + string(os.PathListSeparator) + os.Getenv("PATH"),
|
|
"TMPDIR=" + t.TempDir(),
|
|
}
|
|
output, runErr := cmd.CombinedOutput()
|
|
if runErr == nil {
|
|
t.Fatalf("%s unexpectedly succeeded with failing go list:\n%s", strings.Join(args, " "), output)
|
|
}
|
|
if !strings.Contains(string(output), "injected go list failure") {
|
|
t.Fatalf("%s failure output = %q, want injected failure", strings.Join(args, " "), output)
|
|
}
|
|
}
|
|
}
|
|
|
|
func testPackagePlanRoot(t *testing.T) string {
|
|
t.Helper()
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatalf("resolve repository root: %v", err)
|
|
}
|
|
return root
|
|
}
|
|
|
|
func runTestPackagePlan(t *testing.T, root string, args ...string) string {
|
|
t.Helper()
|
|
script := filepath.Join(root, "scripts", "ci", "test-packages.sh")
|
|
cmd := exec.Command("sh", append([]string{script}, args...)...)
|
|
cmd.Dir = root
|
|
output, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
t.Fatalf("%s %s failed: %v\n%s", script, strings.Join(args, " "), err, output)
|
|
}
|
|
return string(output)
|
|
}
|
|
|
|
func containsPackageSuffix(packages []string, suffix string) bool {
|
|
for _, packagePath := range packages {
|
|
if strings.HasSuffix(packagePath, suffix) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|