616 lines
23 KiB
Go
616 lines
23 KiB
Go
// Copyright 2026 Alibaba Group
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package app
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/cli"
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/corecmd/contract"
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/helpers"
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/pkg/edition"
|
|
"github.com/fatih/color"
|
|
)
|
|
|
|
var (
|
|
agentExamplePlaceholderPattern = regexp.MustCompile(`<([^>]+)>`)
|
|
agentExampleDryRunJSONPattern = regexp.MustCompile(`(?i)"dry_run"\s*:\s*true`)
|
|
agentExampleDryRunPlanPattern = regexp.MustCompile(`(?i)"preview_kind"\s*:\s*"plan"`)
|
|
// agentExampleDryRunInvocationPattern recognizes the executor
|
|
// envelope (and the dev connect preview, which mirrors its shape): the
|
|
// top-level contract is an invocation that merely embeds the would-be
|
|
// request inside response, so it must classify as invocation, not request.
|
|
agentExampleDryRunInvocationPattern = regexp.MustCompile(
|
|
`"kind"\s*:\s*"(?:(?:helper|compat|workflow)_invocation|connect_preview)"`)
|
|
)
|
|
|
|
// TestAgentExamplesContract is the always-on gate. It validates every
|
|
// example, including contract_only entries, against the live bound Cobra path,
|
|
// flags, required arguments, constraints, and final typed safety.
|
|
func TestAgentExamplesContract(t *testing.T) {
|
|
plan := agentExampleExecutionPlan(t)
|
|
if plan.Total == 0 {
|
|
t.Fatal("no reviewed Agent examples were contract validated")
|
|
}
|
|
t.Logf("Agent example contract: total=%d contract=%d dry_run=%d contract_only=%d", plan.Total, plan.Contract, plan.DryRun, plan.ContractOnly)
|
|
}
|
|
|
|
// TestAgentExamplesDryRun first validates every reviewed example against
|
|
// its real BoundCommand, Cobra required arguments, and final typed constraints.
|
|
// It then executes only the deterministic, explicitly declared dry_run subset
|
|
// without injecting --yes. Global flag inheritance is not treated as capability
|
|
// evidence. Runtime failures never create implicit skips. No shell is involved
|
|
// and HOME is isolated.
|
|
func TestAgentExamplesDryRun(t *testing.T) {
|
|
if os.Getenv("DWS_AGENT_EXAMPLES_DRY_RUN") != "1" {
|
|
t.Skip("set DWS_AGENT_EXAMPLES_DRY_RUN=1 to execute the explicitly reviewed Agent dry-run subset")
|
|
}
|
|
|
|
sandboxRoot := t.TempDir()
|
|
homeDir := filepath.Join(sandboxRoot, "home")
|
|
configDir := filepath.Join(sandboxRoot, "config")
|
|
for _, dir := range []string{homeDir, configDir} {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
t.Fatalf("create isolated test directory %s: %v", dir, err)
|
|
}
|
|
}
|
|
setTestHome(t, homeDir)
|
|
t.Setenv("DWS_CONFIG_DIR", configDir)
|
|
t.Setenv("HTTP_PROXY", "http://127.0.0.1:1")
|
|
t.Setenv("HTTPS_PROXY", "http://127.0.0.1:1")
|
|
t.Setenv("NO_PROXY", "")
|
|
|
|
plan := agentExampleExecutionPlan(t)
|
|
if plan.Total == 0 {
|
|
t.Fatal("no reviewed Agent examples were contract validated")
|
|
}
|
|
t.Chdir(sandboxRoot)
|
|
files := newAgentExampleFiles(t, sandboxRoot)
|
|
|
|
selected := 0
|
|
executed := 0
|
|
for _, execution := range plan.Examples {
|
|
if !agentExampleShouldExerciseDryRun(execution) {
|
|
continue
|
|
}
|
|
selected++
|
|
execution := execution
|
|
t.Run(fmt.Sprintf("%s/%d", strings.ReplaceAll(execution.CanonicalPath, ".", "/"), execution.Index), func(t *testing.T) {
|
|
argv, err := cli.ParseAgentExampleArgv(execution.Example)
|
|
if err != nil {
|
|
t.Fatalf("parse example %q: %v", execution.Example, err)
|
|
}
|
|
args := materializeAgentExampleArgv(argv[1:], files)
|
|
if agentExampleHasFlag(args, "yes") {
|
|
t.Fatalf("dry-run gate must not inject or accept --yes\nsource: %s\nargv: %q", execution.Example, args)
|
|
}
|
|
if !agentExampleHasFlag(args, "dry-run") {
|
|
args = append([]string{"--dry-run"}, args...)
|
|
}
|
|
|
|
capture, err := executeAgentExampleCapture(t, args)
|
|
if capture.ToolCallAttempts != 0 {
|
|
t.Fatalf("eligible dry-run attempted %d ToolCaller invocation(s)\nsource: %s\nargv: %q\noutput:\n%s", capture.ToolCallAttempts, execution.Example, args, capture.Output)
|
|
}
|
|
if capture.StdinBytesRead != 0 || agentExamplePromptObserved(capture.Output) {
|
|
t.Fatalf("eligible dry-run entered an interactive confirmation path (stdin bytes read: %d)\nsource: %s\nargv: %q\noutput:\n%s", capture.StdinBytesRead, execution.Example, args, capture.Output)
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("dry-run example failed: %v\nsource: %s\nargv: %q\noutput:\n%s", err, execution.Example, args, capture.Output)
|
|
}
|
|
previewKind, observed := agentExampleDryRunEvidence(capture)
|
|
if !observed {
|
|
t.Fatalf("example returned without audited dry-run evidence (caller dry-run checks: %d)\nsource: %s\nargv: %q\noutput:\n%s", capture.DryRunChecks, execution.Example, args, capture.Output)
|
|
}
|
|
if want := execution.DryRun.PreviewKind; previewKind != want {
|
|
t.Fatalf("dry-run preview kind = %q, Schema declares %q\nsource: %s\nargv: %q\noutput:\n%s", previewKind, want, execution.Example, args, capture.Output)
|
|
}
|
|
t.Logf("dry_run_capability_candidate=%s", previewKind)
|
|
executed++
|
|
})
|
|
}
|
|
if executed != selected {
|
|
t.Fatalf("executed dry_run examples = %d, selected capability set requires %d", executed, selected)
|
|
}
|
|
t.Logf("Agent examples: total=%d contract=%d dry_run_selected=%d planned_dry_run=%d contract_only=%d reviewed_manual=%d", plan.Total, plan.Contract, selected, plan.DryRun, plan.ContractOnly, plan.ReviewedContractOnly)
|
|
reasonCodes := make([]string, 0, len(plan.ContractOnlyByReason))
|
|
for reasonCode := range plan.ContractOnlyByReason {
|
|
reasonCodes = append(reasonCodes, string(reasonCode))
|
|
}
|
|
sort.Strings(reasonCodes)
|
|
for _, reasonCode := range reasonCodes {
|
|
t.Logf("Agent examples contract_only[%s]=%d", reasonCode, plan.ContractOnlyByReason[cli.AgentExampleReasonCode(reasonCode)])
|
|
}
|
|
}
|
|
|
|
// agentExampleShouldExerciseDryRun is the single selection boundary for
|
|
// the runtime gate. Capability comes only from the final typed ToolSpec; the
|
|
// example disposition may narrow that set but can never invent support.
|
|
func agentExampleShouldExerciseDryRun(execution cli.AgentExampleExecution) bool {
|
|
return execution.DryRun != nil && execution.Mode == cli.AgentExampleModeDryRun
|
|
}
|
|
|
|
func agentExampleExecutionPlan(t testing.TB) cli.AgentExampleExecutionPlan {
|
|
t.Helper()
|
|
contractRoot := NewRootCommand()
|
|
effective, err := cli.BuildEffectiveCommandRegistry(contractRoot)
|
|
if err != nil {
|
|
t.Fatalf("BuildEffectiveCommandRegistry() error = %v", err)
|
|
}
|
|
bound, err := cli.BindEffectiveCommandRegistry(contractRoot, effective)
|
|
if err != nil {
|
|
t.Fatalf("BindEffectiveCommandRegistry() error = %v", err)
|
|
}
|
|
registry, err := cli.AssembleSchemaRegistryFromBound(bound)
|
|
if err != nil {
|
|
t.Fatalf("AssembleSchemaRegistryFromBound() error = %v", err)
|
|
}
|
|
if err := cli.ValidateReviewedDryRunCapabilityDelivery(registry); err != nil {
|
|
t.Fatalf("ValidateReviewedDryRunCapabilityDelivery() error = %v", err)
|
|
}
|
|
plan, err := cli.BuildAgentExampleExecutionPlan(bound, registry)
|
|
if err != nil {
|
|
t.Fatalf("BuildAgentExampleExecutionPlan() error = %v", err)
|
|
}
|
|
return plan
|
|
}
|
|
|
|
type agentExampleCapture struct {
|
|
Output string
|
|
DryRunChecks int64
|
|
ToolCallAttempts int64
|
|
StdinBytesRead int64
|
|
}
|
|
|
|
type agentExampleFailClosedCaller struct {
|
|
dryRunChecks atomic.Int64
|
|
toolCallAttempts atomic.Int64
|
|
}
|
|
|
|
func (c *agentExampleFailClosedCaller) CallTool(_ context.Context, productID, toolName string, _ map[string]any) (*edition.ToolResult, error) {
|
|
c.toolCallAttempts.Add(1)
|
|
return nil, fmt.Errorf("real ToolCaller invocation blocked during Agent example dry-run: %s/%s", productID, toolName)
|
|
}
|
|
|
|
func (c *agentExampleFailClosedCaller) Format() string { return "json" }
|
|
|
|
func (c *agentExampleFailClosedCaller) DryRun() bool {
|
|
c.dryRunChecks.Add(1)
|
|
return true
|
|
}
|
|
|
|
func (c *agentExampleFailClosedCaller) Fields() string { return "" }
|
|
func (c *agentExampleFailClosedCaller) JQ() string { return "" }
|
|
|
|
func executeAgentExampleCapture(t testing.TB, args []string) (agentExampleCapture, error) {
|
|
t.Helper()
|
|
oldArgs := os.Args
|
|
os.Args = append([]string{"dws"}, args...)
|
|
defer func() { os.Args = oldArgs }()
|
|
oldStdin := os.Stdin
|
|
promptInput, err := os.CreateTemp(t.TempDir(), "agent-example-stdin-*.txt")
|
|
if err != nil {
|
|
t.Fatalf("open guarded stdin: %v", err)
|
|
}
|
|
defer promptInput.Close()
|
|
if _, err := promptInput.WriteString("no\n"); err != nil {
|
|
t.Fatalf("seed guarded stdin: %v", err)
|
|
}
|
|
if _, err := promptInput.Seek(0, io.SeekStart); err != nil {
|
|
t.Fatalf("rewind guarded stdin: %v", err)
|
|
}
|
|
os.Stdin = promptInput
|
|
defer func() { os.Stdin = oldStdin }()
|
|
|
|
oldStdout, oldStderr := os.Stdout, os.Stderr
|
|
oldColorOutput, oldColorError := color.Output, color.Error
|
|
captureFile, err := os.CreateTemp(t.TempDir(), "agent-example-output-*.log")
|
|
if err != nil {
|
|
t.Fatalf("open output capture file: %v", err)
|
|
}
|
|
defer captureFile.Close()
|
|
os.Stdout, os.Stderr = captureFile, captureFile
|
|
color.Output, color.Error = captureFile, captureFile
|
|
defer func() {
|
|
os.Stdout, os.Stderr = oldStdout, oldStderr
|
|
color.Output, color.Error = oldColorOutput, oldColorError
|
|
}()
|
|
|
|
root := NewRootCommand()
|
|
originalCaller := helpers.GetCaller()
|
|
auditCaller := &agentExampleFailClosedCaller{}
|
|
helpers.InitDeps(auditCaller)
|
|
defer helpers.InitDeps(originalCaller)
|
|
var output bytes.Buffer
|
|
root.SetOut(&output)
|
|
root.SetErr(&output)
|
|
root.SetArgs(args)
|
|
execErr := root.Execute()
|
|
|
|
os.Stdout, os.Stderr = oldStdout, oldStderr
|
|
color.Output, color.Error = oldColorOutput, oldColorError
|
|
if _, err := captureFile.Seek(0, io.SeekStart); err != nil {
|
|
t.Fatalf("rewind output capture file: %v", err)
|
|
}
|
|
captured, readErr := io.ReadAll(captureFile)
|
|
if readErr != nil {
|
|
t.Fatalf("read output capture file: %v", readErr)
|
|
}
|
|
stdinBytesRead, err := promptInput.Seek(0, io.SeekCurrent)
|
|
if err != nil {
|
|
t.Fatalf("inspect guarded stdin: %v", err)
|
|
}
|
|
return agentExampleCapture{
|
|
Output: output.String() + string(captured),
|
|
DryRunChecks: auditCaller.dryRunChecks.Load(),
|
|
ToolCallAttempts: auditCaller.toolCallAttempts.Load(),
|
|
StdinBytesRead: stdinBytesRead,
|
|
}, execErr
|
|
}
|
|
|
|
type agentExampleFiles struct {
|
|
root string
|
|
markdown string
|
|
json string
|
|
batch string
|
|
binary string
|
|
image string
|
|
}
|
|
|
|
func newAgentExampleFiles(t testing.TB, root string) agentExampleFiles {
|
|
t.Helper()
|
|
markdown := filepath.Join(root, "content.md")
|
|
jsonFile := filepath.Join(root, "report.json")
|
|
batch := filepath.Join(root, "styles.json")
|
|
binary := filepath.Join(root, "report.pdf")
|
|
image := filepath.Join(root, "chart.png")
|
|
for path, content := range map[string][]byte{
|
|
markdown: []byte("# Agent dry-run fixture\n\nNo business call is allowed.\n"),
|
|
jsonFile: []byte(`[{"content":"Agent dry-run fixture","sort":"0","key":"fixture","contentType":"markdown","type":"1"}]`),
|
|
batch: []byte(`[{"sheetId":"Sheet1","range":"A1:B2","fontWeight":"bold"}]`),
|
|
binary: []byte("%PDF-1.4\n%%EOF\n"),
|
|
image: {0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'},
|
|
} {
|
|
if err := os.WriteFile(path, content, 0o600); err != nil {
|
|
t.Fatalf("write dry-run fixture %s: %v", path, err)
|
|
}
|
|
}
|
|
return agentExampleFiles{
|
|
root: root,
|
|
markdown: "./" + filepath.Base(markdown),
|
|
json: "./" + filepath.Base(jsonFile),
|
|
batch: "./" + filepath.Base(batch),
|
|
binary: "./" + filepath.Base(binary),
|
|
image: "./" + filepath.Base(image),
|
|
}
|
|
}
|
|
|
|
func materializeAgentExampleArgv(argv []string, files agentExampleFiles) []string {
|
|
result := append([]string(nil), argv...)
|
|
for index := range result {
|
|
result[index] = agentExamplePlaceholderPattern.ReplaceAllStringFunc(result[index], func(match string) string {
|
|
name := strings.TrimSuffix(strings.TrimPrefix(match, "<"), ">")
|
|
switch strings.ToLower(name) {
|
|
case "basetime", "remindertimestamp", "reminder-time-stamp":
|
|
return "1780000000000"
|
|
case "duedateoffset", "due-date-offset":
|
|
return "0"
|
|
case "reminderrules", "reminder-rules":
|
|
return `[{"remindType":"minute","remindTime":10}]`
|
|
case "filepath", "file-path":
|
|
return files.binary
|
|
case "uuid1,uuid2":
|
|
return "uuid1,uuid2"
|
|
default:
|
|
clean := strings.NewReplacer(",", "_", "-", "_", ".", "_").Replace(name)
|
|
return "test_" + clean
|
|
}
|
|
})
|
|
}
|
|
|
|
for index := 0; index < len(result); index++ {
|
|
name, inline, ok := agentExampleLongFlag(result[index])
|
|
if !ok {
|
|
continue
|
|
}
|
|
valueIndex := index + 1
|
|
value := inline
|
|
if inline == "" && valueIndex < len(result) {
|
|
value = result[valueIndex]
|
|
}
|
|
replacement := ""
|
|
switch name {
|
|
case "file", "file-path":
|
|
if strings.Contains(strings.ToLower(value), "png") {
|
|
replacement = files.image
|
|
} else {
|
|
replacement = files.binary
|
|
}
|
|
case "content-file":
|
|
replacement = files.markdown
|
|
case "contents-file":
|
|
replacement = files.json
|
|
case "batch":
|
|
if strings.HasSuffix(strings.ToLower(value), "styles.json") {
|
|
replacement = files.batch
|
|
}
|
|
case "output":
|
|
if value == "." || value == "" {
|
|
replacement = files.root
|
|
} else {
|
|
replacement = filepath.Join(files.root, filepath.Base(value))
|
|
}
|
|
}
|
|
if replacement == "" {
|
|
continue
|
|
}
|
|
if inline != "" {
|
|
result[index] = "--" + name + "=" + replacement
|
|
} else if valueIndex < len(result) {
|
|
result[valueIndex] = replacement
|
|
index++
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func agentExampleLongFlag(argument string) (name, inline string, ok bool) {
|
|
if !strings.HasPrefix(argument, "--") {
|
|
return "", "", false
|
|
}
|
|
name, inline, _ = strings.Cut(strings.TrimPrefix(argument, "--"), "=")
|
|
return name, inline, name != ""
|
|
}
|
|
|
|
func agentExampleHasFlag(argv []string, target string) bool {
|
|
for _, argument := range argv {
|
|
if argument == "--"+target || strings.HasPrefix(argument, "--"+target+"=") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func agentExampleDryRunObserved(capture agentExampleCapture) bool {
|
|
_, ok := agentExampleDryRunEvidence(capture)
|
|
return ok
|
|
}
|
|
|
|
func agentExampleDryRunEvidence(capture agentExampleCapture) (string, bool) {
|
|
normalized := strings.ToLower(capture.Output)
|
|
if agentExampleDryRunJSONPattern.MatchString(capture.Output) && agentExampleDryRunPlanPattern.MatchString(capture.Output) {
|
|
return contract.DryRunPreviewPlan, true
|
|
}
|
|
if agentExampleDryRunJSONPattern.MatchString(capture.Output) && agentExampleDryRunInvocationPattern.MatchString(capture.Output) {
|
|
return contract.DryRunPreviewInvocation, true
|
|
}
|
|
if agentExampleDryRunJSONPattern.MatchString(capture.Output) {
|
|
return contract.DryRunPreviewRequest, true
|
|
}
|
|
if strings.Contains(normalized, "[dry-run]") {
|
|
return contract.DryRunPreviewInvocation, true
|
|
}
|
|
if capture.DryRunChecks > 0 && strings.Contains(capture.Output, "操作:") {
|
|
return contract.DryRunPreviewPlan, true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func TestAgentExampleDryRunEvidenceAcceptsSharedAndCommandPlans(t *testing.T) {
|
|
if !agentExampleDryRunObserved(agentExampleCapture{Output: "[DRY-RUN] Preview only, not executed:\nTool: calendar_list"}) {
|
|
t.Fatal("dry-run output with a Tool and nil Arguments was not recognized")
|
|
}
|
|
if agentExampleDryRunObserved(agentExampleCapture{Output: "Tool: calendar_list"}) {
|
|
t.Fatal("a Tool line without dry-run evidence must not be accepted")
|
|
}
|
|
for _, falseEvidence := range []string{
|
|
"unknown flag: --dry-run",
|
|
"Run again with --dry-run to preview the operation",
|
|
`{"dry_run":false,"executed":true}`,
|
|
} {
|
|
if agentExampleDryRunObserved(agentExampleCapture{Output: falseEvidence}) {
|
|
t.Errorf("non-evidence text was mistaken for a successful dry-run: %q", falseEvidence)
|
|
}
|
|
}
|
|
operationSummary := "操作: 下载钉盘文件\n文件ID: test"
|
|
if agentExampleDryRunObserved(agentExampleCapture{Output: operationSummary}) {
|
|
t.Fatal("a human-only operation summary without an audited dry-run check must not be accepted")
|
|
}
|
|
if !agentExampleDryRunObserved(agentExampleCapture{Output: operationSummary, DryRunChecks: 1}) {
|
|
t.Fatal("a command plan guarded by the injected caller's dry-run check was not recognized")
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageAgentExampleDryRunEvidenceClassifiesStructuredPlan(t *testing.T) {
|
|
kind, observed := agentExampleDryRunEvidence(agentExampleCapture{
|
|
Output: `{"dry_run":true,"executed":false,"preview_kind":"plan"}`,
|
|
DryRunChecks: 1,
|
|
})
|
|
if !observed || kind != contract.DryRunPreviewPlan {
|
|
t.Fatalf("structured plan classified as kind=%q observed=%v", kind, observed)
|
|
}
|
|
if agentExampleDryRunObserved(agentExampleCapture{
|
|
Output: `{"dry_run":false,"executed":false,"preview_kind":"plan"}`,
|
|
DryRunChecks: 1,
|
|
}) {
|
|
t.Fatal("non-dry-run structured plan was accepted as evidence")
|
|
}
|
|
}
|
|
|
|
func TestAgentExampleDryRunEvidenceClassifiesExecutorEnvelopeAsInvocation(t *testing.T) {
|
|
// Executor dry-run envelope: invocation at top level, the would-be request
|
|
// nested inside response — must classify as invocation, never request.
|
|
envelope := `{
|
|
"invocation": {"kind": "helper_invocation", "stage": "helper_override", "implemented": false,
|
|
"dry_run": true, "canonical_product": "devapp", "tool": "create_dev_app"},
|
|
"response": {"dry_run": true, "request": {"jsonrpc": "2.0", "method": "tools/call"},
|
|
"note": "execution skipped by --dry-run"}
|
|
}`
|
|
kind, observed := agentExampleDryRunEvidence(agentExampleCapture{Output: envelope})
|
|
if !observed || kind != contract.DryRunPreviewInvocation {
|
|
t.Fatalf("executor envelope classified as kind=%q observed=%v, want invocation", kind, observed)
|
|
}
|
|
for _, invKind := range []string{"compat_invocation", "workflow_invocation", "connect_preview"} {
|
|
out := strings.Replace(envelope, "helper_invocation", invKind, 1)
|
|
if kind, ok := agentExampleDryRunEvidence(agentExampleCapture{Output: out}); !ok || kind != contract.DryRunPreviewInvocation {
|
|
t.Fatalf("kind %q envelope classified as %q/%v, want invocation", invKind, kind, ok)
|
|
}
|
|
}
|
|
// A bare MCP dry-run document has no invocation kind and stays request.
|
|
mcpDoc := `{"dry_run": true, "executed": false, "tool": "get_dev_app", "arguments": {}}`
|
|
if kind, ok := agentExampleDryRunEvidence(agentExampleCapture{Output: mcpDoc}); !ok || kind != contract.DryRunPreviewRequest {
|
|
t.Fatalf("bare MCP dry-run classified as %q/%v, want request", kind, ok)
|
|
}
|
|
// Plan still wins over an invocation-shaped payload.
|
|
planDoc := `{"dry_run": true, "preview_kind": "plan", "kind": "helper_invocation"}`
|
|
if kind, ok := agentExampleDryRunEvidence(agentExampleCapture{Output: planDoc}); !ok || kind != contract.DryRunPreviewPlan {
|
|
t.Fatalf("plan precedence broken: %q/%v", kind, ok)
|
|
}
|
|
}
|
|
|
|
func agentExamplePromptObserved(output string) bool {
|
|
normalized := strings.ToLower(output)
|
|
for _, marker := range []string{
|
|
"confirm ",
|
|
"confirm deletion?",
|
|
"confirm action?",
|
|
"confirm create?",
|
|
"confirm update?",
|
|
"confirm save?",
|
|
"confirm import?",
|
|
"are you sure",
|
|
"operation cancelled",
|
|
"操作已取消",
|
|
} {
|
|
if strings.Contains(normalized, marker) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestAgentExamplePromptObservedRejectsInteractiveConfirmation(t *testing.T) {
|
|
for _, prompt := range []string{
|
|
"Confirm deletion? (yes/no):",
|
|
"Confirm action? (yes/no):",
|
|
"Confirm create? (yes/no):",
|
|
"Confirm update? (yes/no):",
|
|
"Confirm save? (yes/no):",
|
|
"Confirm import? (yes/no):",
|
|
"Are you sure you want to continue?",
|
|
"Operation cancelled",
|
|
} {
|
|
if !agentExamplePromptObserved(prompt) {
|
|
t.Errorf("interactive confirmation output was not detected: %q", prompt)
|
|
}
|
|
}
|
|
if agentExamplePromptObserved(`{"dry_run":true,"confirmation":"user_required"}`) {
|
|
t.Fatal("typed safety metadata was mistaken for an interactive prompt")
|
|
}
|
|
}
|
|
|
|
func TestAitableAdvpermDisableDryRunSkipsConfirmationAndToolCall(t *testing.T) {
|
|
setTestHome(t, t.TempDir())
|
|
t.Setenv("DWS_CONFIG_DIR", t.TempDir())
|
|
|
|
args := []string{
|
|
"--dry-run", "--format", "json",
|
|
"aitable", "advperm", "disable",
|
|
"--base-id", "BASE_ID",
|
|
}
|
|
if agentExampleHasFlag(args, "yes") {
|
|
t.Fatal("regression test must not bypass confirmation with --yes")
|
|
}
|
|
capture, err := executeAgentExampleCapture(t, args)
|
|
if err != nil {
|
|
t.Fatalf("advperm disable fail-closed dry-run failed: %v\noutput:\n%s", err, capture.Output)
|
|
}
|
|
if capture.StdinBytesRead != 0 || agentExamplePromptObserved(capture.Output) {
|
|
t.Fatalf("advperm disable dry-run entered confirmation (stdin bytes read: %d)\noutput:\n%s", capture.StdinBytesRead, capture.Output)
|
|
}
|
|
if capture.ToolCallAttempts != 0 {
|
|
t.Fatalf("advperm disable dry-run attempted %d real ToolCaller invocation(s)\noutput:\n%s", capture.ToolCallAttempts, capture.Output)
|
|
}
|
|
if !agentExampleDryRunObserved(capture) {
|
|
t.Fatalf("advperm disable returned no audited dry-run evidence (caller dry-run checks: %d)\noutput:\n%s", capture.DryRunChecks, capture.Output)
|
|
}
|
|
}
|
|
|
|
func TestAgentExampleFailClosedCallerRecordsToolCalls(t *testing.T) {
|
|
caller := &agentExampleFailClosedCaller{}
|
|
if !caller.DryRun() {
|
|
t.Fatal("fail-closed caller must advertise dry-run mode")
|
|
}
|
|
if _, err := caller.CallTool(context.Background(), "calendar", "list_events", nil); err == nil {
|
|
t.Fatal("fail-closed caller accepted a ToolCaller invocation")
|
|
}
|
|
if got := caller.dryRunChecks.Load(); got != 1 {
|
|
t.Fatalf("DryRun() checks = %d, want 1", got)
|
|
}
|
|
if got := caller.toolCallAttempts.Load(); got != 1 {
|
|
t.Fatalf("CallTool() attempts = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestAgentExampleChatGroupMuteMemberUsesCommandDryRunPreview(t *testing.T) {
|
|
sandboxRoot := t.TempDir()
|
|
configDir := filepath.Join(sandboxRoot, "config")
|
|
if err := os.MkdirAll(configDir, 0o700); err != nil {
|
|
t.Fatalf("create isolated config directory: %v", err)
|
|
}
|
|
setTestHome(t, sandboxRoot)
|
|
t.Setenv("DWS_CONFIG_DIR", configDir)
|
|
|
|
capture, err := executeAgentExampleCapture(t, []string{
|
|
"--dry-run",
|
|
"chat", "group-mute-member",
|
|
"--group", "test_openConversationId",
|
|
"--users", "userId1,userId2",
|
|
"--mute-time", "3600000",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("group-mute-member dry-run failed: %v\noutput:\n%s", err, capture.Output)
|
|
}
|
|
if capture.ToolCallAttempts != 0 {
|
|
t.Fatalf("group-mute-member dry-run attempted %d ToolCaller invocation(s)\noutput:\n%s", capture.ToolCallAttempts, capture.Output)
|
|
}
|
|
if capture.DryRunChecks == 0 {
|
|
t.Fatalf("group-mute-member did not enter its audited command dry-run path\noutput:\n%s", capture.Output)
|
|
}
|
|
if capture.StdinBytesRead != 0 || agentExamplePromptObserved(capture.Output) {
|
|
t.Fatalf("group-mute-member dry-run entered an interactive prompt (stdin bytes read: %d)\noutput:\n%s", capture.StdinBytesRead, capture.Output)
|
|
}
|
|
if !agentExampleDryRunObserved(capture) {
|
|
t.Fatalf("group-mute-member returned no audited dry-run evidence\noutput:\n%s", capture.Output)
|
|
}
|
|
for _, expected := range []string{`"uids"`, `"userId1"`, `"userId2"`} {
|
|
if !strings.Contains(capture.Output, expected) {
|
|
t.Fatalf("group-mute-member command preview missing %s\noutput:\n%s", expected, capture.Output)
|
|
}
|
|
}
|
|
if strings.Contains(capture.Output, `"openDingTalkIds"`) {
|
|
t.Fatalf("group-mute-member dry-run unexpectedly resolved user IDs remotely\noutput:\n%s", capture.Output)
|
|
}
|
|
}
|