// Copyright 2026 Alibaba Group // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package app import ( "bytes" "context" "fmt" "io" "os" "path/filepath" "regexp" "sort" "strings" "sync/atomic" "testing" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/cli" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/corecmd/contract" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/helpers" "github.com/DingTalk-Real-AI/dingtalk-workspace-cli/pkg/edition" "github.com/fatih/color" ) var ( agentExamplePlaceholderPattern = regexp.MustCompile(`<([^>]+)>`) agentExampleDryRunJSONPattern = regexp.MustCompile(`(?i)"dry_run"\s*:\s*true`) agentExampleDryRunPlanPattern = regexp.MustCompile(`(?i)"preview_kind"\s*:\s*"plan"`) // agentExampleDryRunInvocationPattern recognizes the executor // envelope (and the dev connect preview, which mirrors its shape): the // top-level contract is an invocation that merely embeds the would-be // request inside response, so it must classify as invocation, not request. agentExampleDryRunInvocationPattern = regexp.MustCompile( `"kind"\s*:\s*"(?:(?:helper|compat|workflow)_invocation|connect_preview)"`) ) // TestAgentExamplesContract is the always-on gate. It validates every // example, including contract_only entries, against the live bound Cobra path, // flags, required arguments, constraints, and final typed safety. func TestAgentExamplesContract(t *testing.T) { plan := agentExampleExecutionPlan(t) if plan.Total == 0 { t.Fatal("no reviewed Agent examples were contract validated") } t.Logf("Agent example contract: total=%d contract=%d dry_run=%d contract_only=%d", plan.Total, plan.Contract, plan.DryRun, plan.ContractOnly) } // TestAgentExamplesDryRun first validates every reviewed example against // its real BoundCommand, Cobra required arguments, and final typed constraints. // It then executes only the deterministic, explicitly declared dry_run subset // without injecting --yes. Global flag inheritance is not treated as capability // evidence. Runtime failures never create implicit skips. No shell is involved // and HOME is isolated. func TestAgentExamplesDryRun(t *testing.T) { if os.Getenv("DWS_AGENT_EXAMPLES_DRY_RUN") != "1" { t.Skip("set DWS_AGENT_EXAMPLES_DRY_RUN=1 to execute the explicitly reviewed Agent dry-run subset") } sandboxRoot := t.TempDir() homeDir := filepath.Join(sandboxRoot, "home") configDir := filepath.Join(sandboxRoot, "config") for _, dir := range []string{homeDir, configDir} { if err := os.MkdirAll(dir, 0o700); err != nil { t.Fatalf("create isolated test directory %s: %v", dir, err) } } setTestHome(t, homeDir) t.Setenv("DWS_CONFIG_DIR", configDir) t.Setenv("HTTP_PROXY", "http://127.0.0.1:1") t.Setenv("HTTPS_PROXY", "http://127.0.0.1:1") t.Setenv("NO_PROXY", "") plan := agentExampleExecutionPlan(t) if plan.Total == 0 { t.Fatal("no reviewed Agent examples were contract validated") } t.Chdir(sandboxRoot) files := newAgentExampleFiles(t, sandboxRoot) selected := 0 executed := 0 for _, execution := range plan.Examples { if !agentExampleShouldExerciseDryRun(execution) { continue } selected++ execution := execution t.Run(fmt.Sprintf("%s/%d", strings.ReplaceAll(execution.CanonicalPath, ".", "/"), execution.Index), func(t *testing.T) { argv, err := cli.ParseAgentExampleArgv(execution.Example) if err != nil { t.Fatalf("parse example %q: %v", execution.Example, err) } args := materializeAgentExampleArgv(argv[1:], files) if agentExampleHasFlag(args, "yes") { t.Fatalf("dry-run gate must not inject or accept --yes\nsource: %s\nargv: %q", execution.Example, args) } if !agentExampleHasFlag(args, "dry-run") { args = append([]string{"--dry-run"}, args...) } capture, err := executeAgentExampleCapture(t, args) if capture.ToolCallAttempts != 0 { t.Fatalf("eligible dry-run attempted %d ToolCaller invocation(s)\nsource: %s\nargv: %q\noutput:\n%s", capture.ToolCallAttempts, execution.Example, args, capture.Output) } if capture.StdinBytesRead != 0 || agentExamplePromptObserved(capture.Output) { t.Fatalf("eligible dry-run entered an interactive confirmation path (stdin bytes read: %d)\nsource: %s\nargv: %q\noutput:\n%s", capture.StdinBytesRead, execution.Example, args, capture.Output) } if err != nil { t.Fatalf("dry-run example failed: %v\nsource: %s\nargv: %q\noutput:\n%s", err, execution.Example, args, capture.Output) } previewKind, observed := agentExampleDryRunEvidence(capture) if !observed { t.Fatalf("example returned without audited dry-run evidence (caller dry-run checks: %d)\nsource: %s\nargv: %q\noutput:\n%s", capture.DryRunChecks, execution.Example, args, capture.Output) } if want := execution.DryRun.PreviewKind; previewKind != want { t.Fatalf("dry-run preview kind = %q, Schema declares %q\nsource: %s\nargv: %q\noutput:\n%s", previewKind, want, execution.Example, args, capture.Output) } t.Logf("dry_run_capability_candidate=%s", previewKind) executed++ }) } if executed != selected { t.Fatalf("executed dry_run examples = %d, selected capability set requires %d", executed, selected) } t.Logf("Agent examples: total=%d contract=%d dry_run_selected=%d planned_dry_run=%d contract_only=%d reviewed_manual=%d", plan.Total, plan.Contract, selected, plan.DryRun, plan.ContractOnly, plan.ReviewedContractOnly) reasonCodes := make([]string, 0, len(plan.ContractOnlyByReason)) for reasonCode := range plan.ContractOnlyByReason { reasonCodes = append(reasonCodes, string(reasonCode)) } sort.Strings(reasonCodes) for _, reasonCode := range reasonCodes { t.Logf("Agent examples contract_only[%s]=%d", reasonCode, plan.ContractOnlyByReason[cli.AgentExampleReasonCode(reasonCode)]) } } // agentExampleShouldExerciseDryRun is the single selection boundary for // the runtime gate. Capability comes only from the final typed ToolSpec; the // example disposition may narrow that set but can never invent support. func agentExampleShouldExerciseDryRun(execution cli.AgentExampleExecution) bool { return execution.DryRun != nil && execution.Mode == cli.AgentExampleModeDryRun } func agentExampleExecutionPlan(t testing.TB) cli.AgentExampleExecutionPlan { t.Helper() contractRoot := NewRootCommand() effective, err := cli.BuildEffectiveCommandRegistry(contractRoot) if err != nil { t.Fatalf("BuildEffectiveCommandRegistry() error = %v", err) } bound, err := cli.BindEffectiveCommandRegistry(contractRoot, effective) if err != nil { t.Fatalf("BindEffectiveCommandRegistry() error = %v", err) } registry, err := cli.AssembleSchemaRegistryFromBound(bound) if err != nil { t.Fatalf("AssembleSchemaRegistryFromBound() error = %v", err) } if err := cli.ValidateReviewedDryRunCapabilityDelivery(registry); err != nil { t.Fatalf("ValidateReviewedDryRunCapabilityDelivery() error = %v", err) } plan, err := cli.BuildAgentExampleExecutionPlan(bound, registry) if err != nil { t.Fatalf("BuildAgentExampleExecutionPlan() error = %v", err) } return plan } type agentExampleCapture struct { Output string DryRunChecks int64 ToolCallAttempts int64 StdinBytesRead int64 } type agentExampleFailClosedCaller struct { dryRunChecks atomic.Int64 toolCallAttempts atomic.Int64 } func (c *agentExampleFailClosedCaller) CallTool(_ context.Context, productID, toolName string, _ map[string]any) (*edition.ToolResult, error) { c.toolCallAttempts.Add(1) return nil, fmt.Errorf("real ToolCaller invocation blocked during Agent example dry-run: %s/%s", productID, toolName) } func (c *agentExampleFailClosedCaller) Format() string { return "json" } func (c *agentExampleFailClosedCaller) DryRun() bool { c.dryRunChecks.Add(1) return true } func (c *agentExampleFailClosedCaller) Fields() string { return "" } func (c *agentExampleFailClosedCaller) JQ() string { return "" } func executeAgentExampleCapture(t testing.TB, args []string) (agentExampleCapture, error) { t.Helper() oldArgs := os.Args os.Args = append([]string{"dws"}, args...) defer func() { os.Args = oldArgs }() oldStdin := os.Stdin promptInput, err := os.CreateTemp(t.TempDir(), "agent-example-stdin-*.txt") if err != nil { t.Fatalf("open guarded stdin: %v", err) } defer promptInput.Close() if _, err := promptInput.WriteString("no\n"); err != nil { t.Fatalf("seed guarded stdin: %v", err) } if _, err := promptInput.Seek(0, io.SeekStart); err != nil { t.Fatalf("rewind guarded stdin: %v", err) } os.Stdin = promptInput defer func() { os.Stdin = oldStdin }() oldStdout, oldStderr := os.Stdout, os.Stderr oldColorOutput, oldColorError := color.Output, color.Error captureFile, err := os.CreateTemp(t.TempDir(), "agent-example-output-*.log") if err != nil { t.Fatalf("open output capture file: %v", err) } defer captureFile.Close() os.Stdout, os.Stderr = captureFile, captureFile color.Output, color.Error = captureFile, captureFile defer func() { os.Stdout, os.Stderr = oldStdout, oldStderr color.Output, color.Error = oldColorOutput, oldColorError }() root := NewRootCommand() originalCaller := helpers.GetCaller() auditCaller := &agentExampleFailClosedCaller{} helpers.InitDeps(auditCaller) defer helpers.InitDeps(originalCaller) var output bytes.Buffer root.SetOut(&output) root.SetErr(&output) root.SetArgs(args) execErr := root.Execute() os.Stdout, os.Stderr = oldStdout, oldStderr color.Output, color.Error = oldColorOutput, oldColorError if _, err := captureFile.Seek(0, io.SeekStart); err != nil { t.Fatalf("rewind output capture file: %v", err) } captured, readErr := io.ReadAll(captureFile) if readErr != nil { t.Fatalf("read output capture file: %v", readErr) } stdinBytesRead, err := promptInput.Seek(0, io.SeekCurrent) if err != nil { t.Fatalf("inspect guarded stdin: %v", err) } return agentExampleCapture{ Output: output.String() + string(captured), DryRunChecks: auditCaller.dryRunChecks.Load(), ToolCallAttempts: auditCaller.toolCallAttempts.Load(), StdinBytesRead: stdinBytesRead, }, execErr } type agentExampleFiles struct { root string markdown string json string batch string binary string image string } func newAgentExampleFiles(t testing.TB, root string) agentExampleFiles { t.Helper() markdown := filepath.Join(root, "content.md") jsonFile := filepath.Join(root, "report.json") batch := filepath.Join(root, "styles.json") binary := filepath.Join(root, "report.pdf") image := filepath.Join(root, "chart.png") for path, content := range map[string][]byte{ markdown: []byte("# Agent dry-run fixture\n\nNo business call is allowed.\n"), jsonFile: []byte(`[{"content":"Agent dry-run fixture","sort":"0","key":"fixture","contentType":"markdown","type":"1"}]`), batch: []byte(`[{"sheetId":"Sheet1","range":"A1:B2","fontWeight":"bold"}]`), binary: []byte("%PDF-1.4\n%%EOF\n"), image: {0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}, } { if err := os.WriteFile(path, content, 0o600); err != nil { t.Fatalf("write dry-run fixture %s: %v", path, err) } } return agentExampleFiles{root: root, markdown: markdown, json: jsonFile, batch: batch, binary: binary, image: image} } func materializeAgentExampleArgv(argv []string, files agentExampleFiles) []string { result := append([]string(nil), argv...) for index := range result { result[index] = agentExamplePlaceholderPattern.ReplaceAllStringFunc(result[index], func(match string) string { name := strings.TrimSuffix(strings.TrimPrefix(match, "<"), ">") switch strings.ToLower(name) { case "basetime", "remindertimestamp", "reminder-time-stamp": return "1780000000000" case "duedateoffset", "due-date-offset": return "0" case "reminderrules", "reminder-rules": return `[{"remindType":"minute","remindTime":10}]` case "filepath", "file-path": return files.binary case "uuid1,uuid2": return "uuid1,uuid2" default: clean := strings.NewReplacer(",", "_", "-", "_", ".", "_").Replace(name) return "test_" + clean } }) } for index := 0; index < len(result); index++ { name, inline, ok := agentExampleLongFlag(result[index]) if !ok { continue } valueIndex := index + 1 value := inline if inline == "" && valueIndex < len(result) { value = result[valueIndex] } replacement := "" switch name { case "file", "file-path": if strings.Contains(strings.ToLower(value), "png") { replacement = files.image } else { replacement = files.binary } case "content-file": replacement = files.markdown case "contents-file": replacement = files.json case "batch": if strings.HasSuffix(strings.ToLower(value), "styles.json") { replacement = files.batch } case "output": if value == "." || value == "" { replacement = files.root } else { replacement = filepath.Join(files.root, filepath.Base(value)) } } if replacement == "" { continue } if inline != "" { result[index] = "--" + name + "=" + replacement } else if valueIndex < len(result) { result[valueIndex] = replacement index++ } } return result } func agentExampleLongFlag(argument string) (name, inline string, ok bool) { if !strings.HasPrefix(argument, "--") { return "", "", false } name, inline, _ = strings.Cut(strings.TrimPrefix(argument, "--"), "=") return name, inline, name != "" } func agentExampleHasFlag(argv []string, target string) bool { for _, argument := range argv { if argument == "--"+target || strings.HasPrefix(argument, "--"+target+"=") { return true } } return false } func agentExampleDryRunObserved(capture agentExampleCapture) bool { _, ok := agentExampleDryRunEvidence(capture) return ok } func agentExampleDryRunEvidence(capture agentExampleCapture) (string, bool) { normalized := strings.ToLower(capture.Output) if agentExampleDryRunJSONPattern.MatchString(capture.Output) && agentExampleDryRunPlanPattern.MatchString(capture.Output) { return contract.DryRunPreviewPlan, true } if agentExampleDryRunJSONPattern.MatchString(capture.Output) && agentExampleDryRunInvocationPattern.MatchString(capture.Output) { return contract.DryRunPreviewInvocation, true } if agentExampleDryRunJSONPattern.MatchString(capture.Output) { return contract.DryRunPreviewRequest, true } if strings.Contains(normalized, "[dry-run]") { return contract.DryRunPreviewInvocation, true } if capture.DryRunChecks > 0 && strings.Contains(capture.Output, "操作:") { return contract.DryRunPreviewPlan, true } return "", false } func TestAgentExampleDryRunEvidenceAcceptsSharedAndCommandPlans(t *testing.T) { if !agentExampleDryRunObserved(agentExampleCapture{Output: "[DRY-RUN] Preview only, not executed:\nTool: calendar_list"}) { t.Fatal("dry-run output with a Tool and nil Arguments was not recognized") } if agentExampleDryRunObserved(agentExampleCapture{Output: "Tool: calendar_list"}) { t.Fatal("a Tool line without dry-run evidence must not be accepted") } for _, falseEvidence := range []string{ "unknown flag: --dry-run", "Run again with --dry-run to preview the operation", `{"dry_run":false,"executed":true}`, } { if agentExampleDryRunObserved(agentExampleCapture{Output: falseEvidence}) { t.Errorf("non-evidence text was mistaken for a successful dry-run: %q", falseEvidence) } } operationSummary := "操作: 下载钉盘文件\n文件ID: test" if agentExampleDryRunObserved(agentExampleCapture{Output: operationSummary}) { t.Fatal("a human-only operation summary without an audited dry-run check must not be accepted") } if !agentExampleDryRunObserved(agentExampleCapture{Output: operationSummary, DryRunChecks: 1}) { t.Fatal("a command plan guarded by the injected caller's dry-run check was not recognized") } } func TestCrossPlatformCoverageAgentExampleDryRunEvidenceClassifiesStructuredPlan(t *testing.T) { kind, observed := agentExampleDryRunEvidence(agentExampleCapture{ Output: `{"dry_run":true,"executed":false,"preview_kind":"plan"}`, DryRunChecks: 1, }) if !observed || kind != contract.DryRunPreviewPlan { t.Fatalf("structured plan classified as kind=%q observed=%v", kind, observed) } if agentExampleDryRunObserved(agentExampleCapture{ Output: `{"dry_run":false,"executed":false,"preview_kind":"plan"}`, DryRunChecks: 1, }) { t.Fatal("non-dry-run structured plan was accepted as evidence") } } func TestAgentExampleDryRunEvidenceClassifiesExecutorEnvelopeAsInvocation(t *testing.T) { // Executor dry-run envelope: invocation at top level, the would-be request // nested inside response — must classify as invocation, never request. envelope := `{ "invocation": {"kind": "helper_invocation", "stage": "helper_override", "implemented": false, "dry_run": true, "canonical_product": "devapp", "tool": "create_dev_app"}, "response": {"dry_run": true, "request": {"jsonrpc": "2.0", "method": "tools/call"}, "note": "execution skipped by --dry-run"} }` kind, observed := agentExampleDryRunEvidence(agentExampleCapture{Output: envelope}) if !observed || kind != contract.DryRunPreviewInvocation { t.Fatalf("executor envelope classified as kind=%q observed=%v, want invocation", kind, observed) } for _, invKind := range []string{"compat_invocation", "workflow_invocation", "connect_preview"} { out := strings.Replace(envelope, "helper_invocation", invKind, 1) if kind, ok := agentExampleDryRunEvidence(agentExampleCapture{Output: out}); !ok || kind != contract.DryRunPreviewInvocation { t.Fatalf("kind %q envelope classified as %q/%v, want invocation", invKind, kind, ok) } } // A bare MCP dry-run document has no invocation kind and stays request. mcpDoc := `{"dry_run": true, "executed": false, "tool": "get_dev_app", "arguments": {}}` if kind, ok := agentExampleDryRunEvidence(agentExampleCapture{Output: mcpDoc}); !ok || kind != contract.DryRunPreviewRequest { t.Fatalf("bare MCP dry-run classified as %q/%v, want request", kind, ok) } // Plan still wins over an invocation-shaped payload. planDoc := `{"dry_run": true, "preview_kind": "plan", "kind": "helper_invocation"}` if kind, ok := agentExampleDryRunEvidence(agentExampleCapture{Output: planDoc}); !ok || kind != contract.DryRunPreviewPlan { t.Fatalf("plan precedence broken: %q/%v", kind, ok) } } func agentExamplePromptObserved(output string) bool { normalized := strings.ToLower(output) for _, marker := range []string{ "confirm ", "confirm deletion?", "confirm action?", "confirm create?", "confirm update?", "confirm save?", "confirm import?", "are you sure", "operation cancelled", "操作已取消", } { if strings.Contains(normalized, marker) { return true } } return false } func TestAgentExamplePromptObservedRejectsInteractiveConfirmation(t *testing.T) { for _, prompt := range []string{ "Confirm deletion? (yes/no):", "Confirm action? (yes/no):", "Confirm create? (yes/no):", "Confirm update? (yes/no):", "Confirm save? (yes/no):", "Confirm import? (yes/no):", "Are you sure you want to continue?", "Operation cancelled", } { if !agentExamplePromptObserved(prompt) { t.Errorf("interactive confirmation output was not detected: %q", prompt) } } if agentExamplePromptObserved(`{"dry_run":true,"confirmation":"user_required"}`) { t.Fatal("typed safety metadata was mistaken for an interactive prompt") } } func TestAitableAdvpermDisableDryRunSkipsConfirmationAndToolCall(t *testing.T) { setTestHome(t, t.TempDir()) t.Setenv("DWS_CONFIG_DIR", t.TempDir()) args := []string{ "--dry-run", "--format", "json", "aitable", "advperm", "disable", "--base-id", "BASE_ID", } if agentExampleHasFlag(args, "yes") { t.Fatal("regression test must not bypass confirmation with --yes") } capture, err := executeAgentExampleCapture(t, args) if err != nil { t.Fatalf("advperm disable fail-closed dry-run failed: %v\noutput:\n%s", err, capture.Output) } if capture.StdinBytesRead != 0 || agentExamplePromptObserved(capture.Output) { t.Fatalf("advperm disable dry-run entered confirmation (stdin bytes read: %d)\noutput:\n%s", capture.StdinBytesRead, capture.Output) } if capture.ToolCallAttempts != 0 { t.Fatalf("advperm disable dry-run attempted %d real ToolCaller invocation(s)\noutput:\n%s", capture.ToolCallAttempts, capture.Output) } if !agentExampleDryRunObserved(capture) { t.Fatalf("advperm disable returned no audited dry-run evidence (caller dry-run checks: %d)\noutput:\n%s", capture.DryRunChecks, capture.Output) } } func TestAgentExampleFailClosedCallerRecordsToolCalls(t *testing.T) { caller := &agentExampleFailClosedCaller{} if !caller.DryRun() { t.Fatal("fail-closed caller must advertise dry-run mode") } if _, err := caller.CallTool(context.Background(), "calendar", "list_events", nil); err == nil { t.Fatal("fail-closed caller accepted a ToolCaller invocation") } if got := caller.dryRunChecks.Load(); got != 1 { t.Fatalf("DryRun() checks = %d, want 1", got) } if got := caller.toolCallAttempts.Load(); got != 1 { t.Fatalf("CallTool() attempts = %d, want 1", got) } } func TestAgentExampleChatGroupMuteMemberUsesCommandDryRunPreview(t *testing.T) { sandboxRoot := t.TempDir() configDir := filepath.Join(sandboxRoot, "config") if err := os.MkdirAll(configDir, 0o700); err != nil { t.Fatalf("create isolated config directory: %v", err) } setTestHome(t, sandboxRoot) t.Setenv("DWS_CONFIG_DIR", configDir) capture, err := executeAgentExampleCapture(t, []string{ "--dry-run", "chat", "group-mute-member", "--group", "test_openConversationId", "--users", "userId1,userId2", "--mute-time", "3600000", }) if err != nil { t.Fatalf("group-mute-member dry-run failed: %v\noutput:\n%s", err, capture.Output) } if capture.ToolCallAttempts != 0 { t.Fatalf("group-mute-member dry-run attempted %d ToolCaller invocation(s)\noutput:\n%s", capture.ToolCallAttempts, capture.Output) } if capture.DryRunChecks == 0 { t.Fatalf("group-mute-member did not enter its audited command dry-run path\noutput:\n%s", capture.Output) } if capture.StdinBytesRead != 0 || agentExamplePromptObserved(capture.Output) { t.Fatalf("group-mute-member dry-run entered an interactive prompt (stdin bytes read: %d)\noutput:\n%s", capture.StdinBytesRead, capture.Output) } if !agentExampleDryRunObserved(capture) { t.Fatalf("group-mute-member returned no audited dry-run evidence\noutput:\n%s", capture.Output) } for _, expected := range []string{`"uids"`, `"userId1"`, `"userId2"`} { if !strings.Contains(capture.Output, expected) { t.Fatalf("group-mute-member command preview missing %s\noutput:\n%s", expected, capture.Output) } } if strings.Contains(capture.Output, `"openDingTalkIds"`) { t.Fatalf("group-mute-member dry-run unexpectedly resolved user IDs remotely\noutput:\n%s", capture.Output) } }