445 lines
16 KiB
Go
445 lines
16 KiB
Go
// Copyright 2026 Alibaba Group
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/corecmd/contract"
|
|
)
|
|
|
|
// TestDeliverySchemaCatalogStructure gates the delivered catalog: every tool
|
|
// entry must conform to the unified closed structure. If this fails after
|
|
// regeneration, either fix the generator inputs or deliberately extend the
|
|
// whitelist in schema_catalog_structure.go.
|
|
func TestDeliverySchemaCatalogStructure(t *testing.T) {
|
|
// After PR #656 the catalog is embedded as per-product shards, not a single
|
|
// JSON file. The loaded snapshot is the reassembled result — serialize it
|
|
// back to JSON and validate the same closed structure.
|
|
loaded := mustDeliverySchemaCatalogMaps(t)
|
|
data, err := json.Marshal(loaded.Snapshot)
|
|
if err != nil {
|
|
t.Fatalf("marshal delivery catalog snapshot: %v", err)
|
|
}
|
|
if err := ValidateCatalogStructure(data); err != nil {
|
|
t.Fatalf("delivery schema catalog violates the unified command structure: %v", err)
|
|
}
|
|
}
|
|
|
|
func validCatalogToolEntry() map[string]any {
|
|
return map[string]any{
|
|
"agent_metadata_source": ProvenanceEmbeddedSkillMetadata,
|
|
"agent_source_refs": []any{"skills/mono/references/products/aitable.md"},
|
|
"agent_summary": "创建多维表格记录",
|
|
"agent_summary_source": "dws-agent-selection/aitable",
|
|
"availability": "available",
|
|
"avoid_when": []any{"批量更新时请用 record update"},
|
|
"canonical_path": "aitable.record_create",
|
|
"cli_name": "create",
|
|
"cli_path": "aitable record create",
|
|
"confirmation": "not_required",
|
|
"description": "在数据表中创建一条或多条记录。",
|
|
"display": "多维表格",
|
|
"effect": "write",
|
|
"effect_source": "command-verb",
|
|
"examples": []any{"dws aitable record create --base-id x"},
|
|
"field_provenance": map[string]any{"effect": map[string]any{"precedence": "reviewed_explicit"}},
|
|
"has_parameters": true,
|
|
"idempotency": "non_idempotent",
|
|
"interface_mode": "mcp",
|
|
"interface_ref": map[string]any{"product_id": "aitable", "rpc_name": "create_records"},
|
|
"is_alias": false,
|
|
"name": "record create",
|
|
"parameter_count": float64(1),
|
|
"parameters": map[string]any{
|
|
"base-id": map[string]any{
|
|
"type": "string",
|
|
"description": "目标多维表格 ID。",
|
|
"required": true,
|
|
"field_provenance": map[string]any{"description": map[string]any{"precedence": "reviewed_explicit"}},
|
|
},
|
|
},
|
|
"path": "aitable.record_create",
|
|
"primary_cli_path": "aitable record create",
|
|
"product_id": "aitable",
|
|
"reviewed": true,
|
|
"risk": "medium",
|
|
"source": "cobra+registry",
|
|
"title": "创建记录",
|
|
"use_when": []any{"需要向多维表格写入新记录时"},
|
|
}
|
|
}
|
|
|
|
func catalogPayload(t *testing.T, entry map[string]any) []byte {
|
|
t.Helper()
|
|
payload, err := json.Marshal(map[string]any{
|
|
"version": SchemaCatalogSnapshotVersion,
|
|
"tools": map[string]any{"aitable.record_create": entry},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal payload: %v", err)
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func TestValidateCatalogStructureAcceptsValidEntry(t *testing.T) {
|
|
if err := ValidateCatalogStructure(catalogPayload(t, validCatalogToolEntry())); err != nil {
|
|
t.Fatalf("ValidateCatalogStructure() error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateCatalogStructureAcceptsOptionalResultObject(t *testing.T) {
|
|
entry := validCatalogToolEntry()
|
|
entry["result"] = map[string]any{
|
|
"outcomes": []any{"success", "failure"},
|
|
"data_schema": map[string]any{"type": "object"},
|
|
}
|
|
if err := ValidateCatalogStructure(catalogPayload(t, entry)); err != nil {
|
|
t.Fatalf("ValidateCatalogStructure() error = %v", err)
|
|
}
|
|
entry["result"] = "invalid"
|
|
if err := ValidateCatalogStructure(catalogPayload(t, entry)); err == nil || !strings.Contains(err.Error(), `field "result" must be an object`) {
|
|
t.Fatalf("invalid result error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateCatalogStructureAcceptsStandalonePagination(t *testing.T) {
|
|
entry := validCatalogToolEntry()
|
|
parameters := entry["parameters"].(map[string]any)
|
|
parameters["cursor"] = map[string]any{
|
|
"description": "续页游标",
|
|
"field_provenance": map[string]any{},
|
|
"required": false,
|
|
"type": "string",
|
|
}
|
|
entry["parameter_count"] = float64(len(parameters))
|
|
entry["has_parameters"] = true
|
|
entry["pagination"] = map[string]any{
|
|
"kind": contract.PaginationKindCursor,
|
|
"cursor_parameter": "cursor",
|
|
"meta_path": contract.PaginationMetaPath,
|
|
"endpoint_exhausted_path": contract.PaginationExhaustedPath,
|
|
"next_token_path": contract.PaginationNextTokenPath,
|
|
}
|
|
if err := ValidateCatalogStructure(catalogPayload(t, entry)); err != nil {
|
|
t.Fatalf("ValidateCatalogStructure() error = %v", err)
|
|
}
|
|
|
|
entry["pagination"].(map[string]any)["next_token_path"] = "data.nextCursor"
|
|
if err := ValidateCatalogStructure(catalogPayload(t, entry)); err == nil || !strings.Contains(err.Error(), "next_token_path") {
|
|
t.Fatalf("invalid pagination error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateCatalogStructureRejectsMalformedStandalonePagination(t *testing.T) {
|
|
validPaginationEntry := func() map[string]any {
|
|
entry := validCatalogToolEntry()
|
|
parameters := entry["parameters"].(map[string]any)
|
|
parameters["cursor"] = map[string]any{
|
|
"description": "续页游标",
|
|
"field_provenance": map[string]any{},
|
|
"required": false,
|
|
"type": "string",
|
|
}
|
|
entry["parameter_count"] = float64(len(parameters))
|
|
entry["pagination"] = map[string]any{
|
|
"kind": contract.PaginationKindCursor,
|
|
"cursor_parameter": "cursor",
|
|
"meta_path": contract.PaginationMetaPath,
|
|
"endpoint_exhausted_path": contract.PaginationExhaustedPath,
|
|
"next_token_path": contract.PaginationNextTokenPath,
|
|
}
|
|
return entry
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
mutate func(map[string]any)
|
|
want string
|
|
}{
|
|
{"not an object", func(entry map[string]any) { entry["pagination"] = "cursor" }, `field "pagination" must be an object`},
|
|
{"empty cursor", func(entry map[string]any) { entry["pagination"].(map[string]any)["cursor_parameter"] = " " }, "cursor_parameter must be a non-empty string"},
|
|
{"unknown cursor", func(entry map[string]any) { entry["pagination"].(map[string]any)["cursor_parameter"] = "page-token" }, "references missing parameter"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
entry := validPaginationEntry()
|
|
tc.mutate(entry)
|
|
err := ValidateCatalogStructure(catalogPayload(t, entry))
|
|
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
|
t.Fatalf("ValidateCatalogStructure() error = %v, want %q", err, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageValidateCatalogStructureRejectsViolations(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
mutate func(entry map[string]any)
|
|
want string
|
|
}{
|
|
{
|
|
name: "unknown tool field",
|
|
mutate: func(e map[string]any) { e["surprise"] = "x" },
|
|
want: `unknown field "surprise"`,
|
|
},
|
|
{
|
|
name: "missing required core field",
|
|
mutate: func(e map[string]any) { delete(e, "agent_summary") },
|
|
want: `"agent_summary" must be a non-empty string`,
|
|
},
|
|
{
|
|
name: "bad effect enum",
|
|
mutate: func(e map[string]any) { e["effect"] = "mutate" },
|
|
want: `"effect" = "mutate"`,
|
|
},
|
|
{
|
|
// 非字符串枚举字段：类型错误由字符串校验报告，枚举循环跳过。
|
|
name: "non-string enum field skips enum check",
|
|
mutate: func(e map[string]any) { e["effect"] = float64(1) },
|
|
want: `"effect" must be a non-empty string`,
|
|
},
|
|
{
|
|
name: "mcp without interface_ref",
|
|
mutate: func(e map[string]any) { delete(e, "interface_ref") },
|
|
want: "interface_mode=mcp requires interface_ref",
|
|
},
|
|
{
|
|
name: "composite without interface_reason",
|
|
mutate: func(e map[string]any) {
|
|
e["interface_mode"] = "composite"
|
|
delete(e, "interface_ref")
|
|
},
|
|
want: "interface_mode=composite requires a non-empty interface_reason",
|
|
},
|
|
{
|
|
name: "composite must not keep interface_ref",
|
|
mutate: func(e map[string]any) {
|
|
e["interface_mode"] = "composite"
|
|
e["interface_reason"] = "本地组合多个 MCP 调用"
|
|
},
|
|
want: "interface_mode=composite must not set interface_ref",
|
|
},
|
|
{
|
|
name: "parameter_count mismatch",
|
|
mutate: func(e map[string]any) { e["parameter_count"] = float64(2) },
|
|
want: "parameter_count = 2, want 1",
|
|
},
|
|
{
|
|
name: "has_parameters mismatch",
|
|
mutate: func(e map[string]any) { e["has_parameters"] = false },
|
|
want: "has_parameters = false, want true",
|
|
},
|
|
{
|
|
name: "param missing required",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
delete(params["base-id"].(map[string]any), "required")
|
|
},
|
|
want: `parameter "base-id": required must be a boolean`,
|
|
},
|
|
{
|
|
name: "param unknown field",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["mystery"] = true
|
|
},
|
|
want: `parameter "base-id": unknown field "mystery"`,
|
|
},
|
|
{
|
|
name: "param bad type enum",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["type"] = "text"
|
|
},
|
|
want: `parameter "base-id": type = "text"`,
|
|
},
|
|
{
|
|
name: "param default must be string",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["default"] = float64(3)
|
|
},
|
|
want: `parameter "base-id": default must be a string`,
|
|
},
|
|
{
|
|
name: "reviewed must be bool",
|
|
mutate: func(e map[string]any) { e["reviewed"] = "yes" },
|
|
want: `"reviewed" must be a boolean`,
|
|
},
|
|
{
|
|
name: "examples must be string array",
|
|
mutate: func(e map[string]any) { e["examples"] = []any{"ok", float64(1)} },
|
|
want: `"examples" must be an array of strings`,
|
|
},
|
|
{
|
|
name: "use_when must be an array at all",
|
|
mutate: func(e map[string]any) { e["use_when"] = "单个字符串" },
|
|
want: `"use_when" must be an array of strings`,
|
|
},
|
|
{
|
|
name: "field_provenance must be object",
|
|
mutate: func(e map[string]any) { e["field_provenance"] = "not-an-object" },
|
|
want: `"field_provenance" must be an object`,
|
|
},
|
|
{
|
|
name: "parameters must be object",
|
|
mutate: func(e map[string]any) { e["parameters"] = []any{"x"} },
|
|
want: `"parameters" must be an object`,
|
|
},
|
|
{
|
|
name: "parameter_count must be number",
|
|
mutate: func(e map[string]any) { e["parameter_count"] = "1" },
|
|
want: `"parameter_count" must be a number`,
|
|
},
|
|
{
|
|
name: "parameter entry must be object",
|
|
mutate: func(e map[string]any) {
|
|
e["parameters"] = map[string]any{"base-id": "not-an-object"}
|
|
e["parameter_count"] = float64(1)
|
|
},
|
|
want: `parameter "base-id" must be an object`,
|
|
},
|
|
{
|
|
name: "interface_ref must be object",
|
|
mutate: func(e map[string]any) { e["interface_ref"] = "aitable.create_records" },
|
|
want: "interface_ref must be an object",
|
|
},
|
|
{
|
|
name: "interface_ref keys must be non-empty",
|
|
mutate: func(e map[string]any) {
|
|
e["interface_ref"] = map[string]any{"product_id": " ", "rpc_name": float64(1)}
|
|
},
|
|
want: "interface_ref.product_id must be a non-empty string",
|
|
},
|
|
{
|
|
name: "mcp must not set interface_reason",
|
|
mutate: func(e map[string]any) { e["interface_reason"] = "多余理由" },
|
|
want: "interface_mode=mcp must not set interface_reason",
|
|
},
|
|
{
|
|
name: "param description must be non-empty",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["description"] = " "
|
|
},
|
|
want: `parameter "base-id": description must be a non-empty string`,
|
|
},
|
|
{
|
|
name: "param field_provenance must be object",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["field_provenance"] = "x"
|
|
},
|
|
want: `parameter "base-id": field_provenance must be an object`,
|
|
},
|
|
{
|
|
name: "param cli_required must be bool",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["cli_required"] = "yes"
|
|
},
|
|
want: `parameter "base-id": cli_required must be a boolean`,
|
|
},
|
|
{
|
|
name: "param enum must be array",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["enum"] = "A,B"
|
|
},
|
|
want: `parameter "base-id": enum must be an array`,
|
|
},
|
|
{
|
|
name: "param enum items must be strings",
|
|
mutate: func(e map[string]any) {
|
|
params := e["parameters"].(map[string]any)
|
|
params["base-id"].(map[string]any)["enum"] = []any{"A", float64(2)}
|
|
},
|
|
want: `parameter "base-id": enum[1] must be a string`,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
entry := validCatalogToolEntry()
|
|
tc.mutate(entry)
|
|
err := ValidateCatalogStructure(catalogPayload(t, entry))
|
|
if err == nil {
|
|
t.Fatalf("ValidateCatalogStructure() = nil, want violation containing %q", tc.want)
|
|
}
|
|
if !strings.Contains(err.Error(), tc.want) {
|
|
t.Fatalf("ValidateCatalogStructure() error = %v, want substring %q", err, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageValidateCatalogStructureRejectsBadEnvelope(t *testing.T) {
|
|
if err := ValidateCatalogStructure([]byte(`{"version":99,"tools":{}}`)); err == nil {
|
|
t.Fatal("ValidateCatalogStructure() = nil, want version error")
|
|
}
|
|
if err := ValidateCatalogStructure([]byte(`{"version":1,"tools":{}}`)); err == nil {
|
|
t.Fatal("ValidateCatalogStructure() = nil, want empty-tools error")
|
|
}
|
|
if err := ValidateCatalogStructure([]byte(`not json`)); err == nil {
|
|
t.Fatal("ValidateCatalogStructure() = nil, want decode error")
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageValidateCatalogStructureSortsViolationsAcrossTools(t *testing.T) {
|
|
// 两个工具各制造两条违规，驱动排序比较器的 tool 与 message 两个分支。
|
|
bad1 := validCatalogToolEntry()
|
|
bad1["effect"] = "mutate"
|
|
bad1["risk"] = "extreme"
|
|
bad2 := validCatalogToolEntry()
|
|
bad2["canonical_path"] = "zz.last_tool"
|
|
bad2["effect"] = "mutate"
|
|
bad2["risk"] = "extreme"
|
|
payload, err := json.Marshal(map[string]any{
|
|
"version": SchemaCatalogSnapshotVersion,
|
|
"tools": map[string]any{"zz.last_tool": bad2, "aitable.record_create": bad1},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
verr := ValidateCatalogStructure(payload)
|
|
if verr == nil {
|
|
t.Fatal("ValidateCatalogStructure() = nil, want sorted violations")
|
|
}
|
|
msg := verr.Error()
|
|
if !strings.Contains(msg, "(4 total)") {
|
|
t.Fatalf("error = %v, want 4 total violations", verr)
|
|
}
|
|
if first, second := strings.Index(msg, "aitable.record_create"), strings.Index(msg, "zz.last_tool"); first < 0 || second < 0 || first > second {
|
|
t.Fatalf("violations not sorted by tool: %v", verr)
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageValidateCatalogStructureTruncatesViolationList(t *testing.T) {
|
|
// 单工具制造超过 25 条违规，验证截断提示。
|
|
entry := validCatalogToolEntry()
|
|
for i := 0; i < 30; i++ {
|
|
entry[fmt.Sprintf("unknown_field_%02d", i)] = true
|
|
}
|
|
err := ValidateCatalogStructure(catalogPayload(t, entry))
|
|
if err == nil {
|
|
t.Fatal("ValidateCatalogStructure() = nil, want truncated violation list")
|
|
}
|
|
if !strings.Contains(err.Error(), "more violations") {
|
|
t.Fatalf("error = %v, want truncation marker", err)
|
|
}
|
|
}
|