fix(fetch-mcp-metadata): count registry stubs as unmatched in coverage
matched_tools claimed every surface tool matched even when entries were registry stubs with no live MCP metadata, and unmatched_tools was hardcoded to 0. Coverage now excludes stubs from matched_tools, reports them as unmatched, and a registry JSON parse failure warns instead of silently producing a stub-only snapshot. The schema catalog policy invariant is relaxed to match the honest accounting.
This commit is contained in:
1 parent
eb2658ca68
commit
bbdf843ef3
3 files changed
+26
-10
No files matched your search
@@ -163,7 +163,7 @@ func run(args []string, stderr io.Writer) int {
|
||||
metadata := map[string]any{
|
||||
"version": 1,
|
||||
"source": "mcp-tools-list+cli-registry",
|
||||
"coverage": buildCoverage(len(servers), failedServices, totalRaw, len(allTools)),
|
||||
"coverage": buildCoverage(len(servers), failedServices, totalRaw, len(allTools), stubs),
|
||||
"tools": allTools,
|
||||
}
|
||||
|
||||
@@ -209,9 +209,10 @@ func writeMetadata(path string, metadata map[string]any) error {
|
||||
}
|
||||
|
||||
// buildCoverage reports snapshot coverage honestly: snapshot_services only
|
||||
// counts services whose tools/list succeeded, and missing_services names the
|
||||
// failures so downstream policy checks can detect an incomplete snapshot.
|
||||
func buildCoverage(sourceServices int, failedServices []string, sourceTools, surfaceTools int) map[string]any {
|
||||
// counts services whose tools/list succeeded, missing_services names the
|
||||
// failures, and matched_tools excludes registry stubs (entries carrying no
|
||||
// live MCP metadata) so a stub-heavy snapshot cannot claim full matching.
|
||||
func buildCoverage(sourceServices int, failedServices []string, sourceTools, surfaceTools, stubs int) map[string]any {
|
||||
missing := failedServices
|
||||
if missing == nil {
|
||||
missing = []string{}
|
||||
@@ -223,9 +224,9 @@ func buildCoverage(sourceServices int, failedServices []string, sourceTools, sur
|
||||
"missing_services": missing,
|
||||
"source_tools": sourceTools,
|
||||
"surface_tools": surfaceTools,
|
||||
"matched_tools": surfaceTools,
|
||||
"matched_tools": surfaceTools - stubs,
|
||||
"aliased_tools": 0,
|
||||
"unmatched_tools": 0,
|
||||
"unmatched_tools": stubs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,6 +271,9 @@ func loadRegistryInterfaceRefs(stderr io.Writer) map[string]map[string]string {
|
||||
} `json:"products"`
|
||||
}
|
||||
if err := json.Unmarshal(data, ®); err != nil {
|
||||
// 与读文件失败同等告警:静默返回空映射会让所有 live tool 被丢弃、
|
||||
// 产出 stub-only 快照且零提示(P1#1 的故障模式)。
|
||||
fmt.Fprintf(stderr, "fetch_mcp_metadata: warning: cannot parse registry: %v\n", err)
|
||||
return map[string]map[string]string{}
|
||||
}
|
||||
out := make(map[string]map[string]string)
|
||||
|
||||
@@ -94,7 +94,7 @@ func TestMergeLiveMCPToolRefreshesExistingMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildCoverageReportsFailedServices(t *testing.T) {
|
||||
got := buildCoverage(26, []string{"doc", "sheet"}, 800, 813)
|
||||
got := buildCoverage(26, []string{"doc", "sheet"}, 800, 813, 40)
|
||||
if got["source_services"] != 26 {
|
||||
t.Fatalf("source_services = %v, want 26", got["source_services"])
|
||||
}
|
||||
@@ -104,19 +104,26 @@ func TestBuildCoverageReportsFailedServices(t *testing.T) {
|
||||
if !reflect.DeepEqual(got["missing_services"], []string{"doc", "sheet"}) {
|
||||
t.Fatalf("missing_services = %#v, want failed service IDs", got["missing_services"])
|
||||
}
|
||||
if got["source_tools"] != 800 || got["surface_tools"] != 813 || got["matched_tools"] != 813 {
|
||||
// matched 必须剔除 stub 占位,unmatched 据实等于 stub 数。
|
||||
if got["matched_tools"] != 773 || got["unmatched_tools"] != 40 {
|
||||
t.Fatalf("matched/unmatched = %v/%v, want 773/40 (813 surface - 40 stubs)", got["matched_tools"], got["unmatched_tools"])
|
||||
}
|
||||
if got["source_tools"] != 800 || got["surface_tools"] != 813 {
|
||||
t.Fatalf("tool counts = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCoverageFullSnapshotHasNoMissingServices(t *testing.T) {
|
||||
got := buildCoverage(26, nil, 813, 813)
|
||||
got := buildCoverage(26, nil, 813, 813, 0)
|
||||
if got["snapshot_services"] != 26 {
|
||||
t.Fatalf("snapshot_services = %v, want 26", got["snapshot_services"])
|
||||
}
|
||||
if !reflect.DeepEqual(got["missing_services"], []string{}) {
|
||||
t.Fatalf("missing_services = %#v, want empty non-nil slice", got["missing_services"])
|
||||
}
|
||||
if got["matched_tools"] != 813 || got["unmatched_tools"] != 0 {
|
||||
t.Fatalf("matched/unmatched = %v/%v, want 813/0 for stub-free snapshot", got["matched_tools"], got["unmatched_tools"])
|
||||
}
|
||||
}
|
||||
|
||||
// fakeLister returns canned tools/list results per endpoint.
|
||||
@@ -310,6 +317,10 @@ func TestRunUnparsableRegistryYieldsNoRefs(t *testing.T) {
|
||||
if refs := loadRegistryInterfaceRefs(&stderr); len(refs) != 0 {
|
||||
t.Fatalf("refs = %v, want empty for unparsable registry", refs)
|
||||
}
|
||||
// 解析失败必须有告警,不得静默产出空映射。
|
||||
if !strings.Contains(stderr.String(), "cannot parse registry") {
|
||||
t.Fatalf("stderr = %q, want parse warning", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunWriteFailure(t *testing.T) {
|
||||
|
||||
@@ -47,7 +47,8 @@ if ! jq -e '
|
||||
.coverage.source_services == (.coverage.snapshot_services + (.coverage.missing_services | length)) and
|
||||
.coverage.surface_tools == (.coverage.matched_tools + .coverage.unmatched_tools) and
|
||||
.coverage.source_tools >= .coverage.surface_tools and
|
||||
.coverage.matched_tools == (.tools | length) and
|
||||
.coverage.matched_tools <= (.tools | length) and
|
||||
(.tools | length) <= .coverage.surface_tools and
|
||||
.coverage.aliased_tools <= .coverage.matched_tools
|
||||
' internal/cli/schema_mcp_metadata.json >/dev/null; then
|
||||
printf 'MCP source-revision snapshot coverage is inconsistent: snapshot_registry=%s\n' \
|
||||
|
||||
Reference in new issue
Block a user