575 lines
21 KiB
Go
575 lines
21 KiB
Go
// Copyright 2026 Alibaba Group
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package app
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/cli"
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/helpers"
|
|
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/pkg/edition"
|
|
"github.com/fatih/color"
|
|
)
|
|
|
|
var (
|
|
manualAgentExamplePlaceholderPattern = regexp.MustCompile(`<([^>]+)>`)
|
|
manualAgentExampleDryRunJSONPattern = regexp.MustCompile(`(?i)"dry_run"\s*:\s*true`)
|
|
manualAgentExampleDryRunPlanPattern = regexp.MustCompile(`(?i)"preview_kind"\s*:\s*"plan"`)
|
|
)
|
|
|
|
// TestManualAgentExamplesContract is the always-on gate. It validates every
|
|
// example, including contract_only entries, against the live bound Cobra path,
|
|
// flags, required arguments, constraints, and final typed safety.
|
|
func TestManualAgentExamplesContract(t *testing.T) {
|
|
plan := manualAgentExampleExecutionPlan(t)
|
|
if plan.Total == 0 {
|
|
t.Fatal("no reviewed Agent examples were contract validated")
|
|
}
|
|
t.Logf("Agent example contract: total=%d contract=%d dry_run=%d contract_only=%d", plan.Total, plan.Contract, plan.DryRun, plan.ContractOnly)
|
|
}
|
|
|
|
// TestManualAgentExamplesDryRun first validates every reviewed example against
|
|
// its real BoundCommand, Cobra required arguments, and final typed constraints.
|
|
// It then executes only the deterministic, explicitly declared dry_run subset
|
|
// without injecting --yes. Global flag inheritance is not treated as capability
|
|
// evidence. Runtime failures never create implicit skips. No shell is involved
|
|
// and HOME is isolated.
|
|
func TestManualAgentExamplesDryRun(t *testing.T) {
|
|
if os.Getenv("DWS_AGENT_EXAMPLES_DRY_RUN") != "1" {
|
|
t.Skip("set DWS_AGENT_EXAMPLES_DRY_RUN=1 to execute the explicitly reviewed Agent dry-run subset")
|
|
}
|
|
|
|
sandboxRoot := t.TempDir()
|
|
homeDir := filepath.Join(sandboxRoot, "home")
|
|
configDir := filepath.Join(sandboxRoot, "config")
|
|
for _, dir := range []string{homeDir, configDir} {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
t.Fatalf("create isolated test directory %s: %v", dir, err)
|
|
}
|
|
}
|
|
setTestHome(t, homeDir)
|
|
t.Setenv("DWS_CONFIG_DIR", configDir)
|
|
t.Setenv("HTTP_PROXY", "http://127.0.0.1:1")
|
|
t.Setenv("HTTPS_PROXY", "http://127.0.0.1:1")
|
|
t.Setenv("NO_PROXY", "")
|
|
|
|
plan := manualAgentExampleExecutionPlan(t)
|
|
if plan.Total == 0 {
|
|
t.Fatal("no reviewed Agent examples were contract validated")
|
|
}
|
|
t.Chdir(sandboxRoot)
|
|
files := newManualAgentExampleFiles(t, sandboxRoot)
|
|
|
|
selected := 0
|
|
executed := 0
|
|
for _, execution := range plan.Examples {
|
|
if !manualAgentExampleShouldExerciseDryRun(execution) {
|
|
continue
|
|
}
|
|
selected++
|
|
execution := execution
|
|
t.Run(fmt.Sprintf("%s/%d", strings.ReplaceAll(execution.CanonicalPath, ".", "/"), execution.Index), func(t *testing.T) {
|
|
argv, err := cli.ParseManualAgentExampleArgv(execution.Example)
|
|
if err != nil {
|
|
t.Fatalf("parse example %q: %v", execution.Example, err)
|
|
}
|
|
args := materializeManualAgentExampleArgv(argv[1:], files)
|
|
if manualAgentExampleHasFlag(args, "yes") {
|
|
t.Fatalf("dry-run gate must not inject or accept --yes\nsource: %s\nargv: %q", execution.Example, args)
|
|
}
|
|
if !manualAgentExampleHasFlag(args, "dry-run") {
|
|
args = append([]string{"--dry-run"}, args...)
|
|
}
|
|
|
|
capture, err := executeManualAgentExampleCapture(t, args)
|
|
if capture.ToolCallAttempts != 0 {
|
|
t.Fatalf("eligible dry-run attempted %d ToolCaller invocation(s)\nsource: %s\nargv: %q\noutput:\n%s", capture.ToolCallAttempts, execution.Example, args, capture.Output)
|
|
}
|
|
if capture.StdinBytesRead != 0 || manualAgentExamplePromptObserved(capture.Output) {
|
|
t.Fatalf("eligible dry-run entered an interactive confirmation path (stdin bytes read: %d)\nsource: %s\nargv: %q\noutput:\n%s", capture.StdinBytesRead, execution.Example, args, capture.Output)
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("dry-run example failed: %v\nsource: %s\nargv: %q\noutput:\n%s", err, execution.Example, args, capture.Output)
|
|
}
|
|
previewKind, observed := manualAgentExampleDryRunEvidence(capture)
|
|
if !observed {
|
|
t.Fatalf("example returned without audited dry-run evidence (caller dry-run checks: %d)\nsource: %s\nargv: %q\noutput:\n%s", capture.DryRunChecks, execution.Example, args, capture.Output)
|
|
}
|
|
if want := execution.DryRun.PreviewKind; previewKind != want {
|
|
t.Fatalf("dry-run preview kind = %q, Schema declares %q\nsource: %s\nargv: %q\noutput:\n%s", previewKind, want, execution.Example, args, capture.Output)
|
|
}
|
|
t.Logf("dry_run_capability_candidate=%s", previewKind)
|
|
executed++
|
|
})
|
|
}
|
|
if executed != selected {
|
|
t.Fatalf("executed dry_run examples = %d, selected capability set requires %d", executed, selected)
|
|
}
|
|
t.Logf("Agent examples: total=%d contract=%d dry_run_selected=%d planned_dry_run=%d contract_only=%d reviewed_manual=%d", plan.Total, plan.Contract, selected, plan.DryRun, plan.ContractOnly, plan.ReviewedContractOnly)
|
|
reasonCodes := make([]string, 0, len(plan.ContractOnlyByReason))
|
|
for reasonCode := range plan.ContractOnlyByReason {
|
|
reasonCodes = append(reasonCodes, string(reasonCode))
|
|
}
|
|
sort.Strings(reasonCodes)
|
|
for _, reasonCode := range reasonCodes {
|
|
t.Logf("Agent examples contract_only[%s]=%d", reasonCode, plan.ContractOnlyByReason[cli.ManualAgentExampleReasonCode(reasonCode)])
|
|
}
|
|
}
|
|
|
|
// manualAgentExampleShouldExerciseDryRun is the single selection boundary for
|
|
// the runtime gate. Capability comes only from the final typed ToolSpec; the
|
|
// example disposition may narrow that set but can never invent support.
|
|
func manualAgentExampleShouldExerciseDryRun(execution cli.ManualAgentExampleExecution) bool {
|
|
return execution.DryRun != nil && execution.Mode == cli.ManualAgentExampleModeDryRun
|
|
}
|
|
|
|
func manualAgentExampleExecutionPlan(t testing.TB) cli.ManualAgentExampleExecutionPlan {
|
|
t.Helper()
|
|
hints, err := cli.LoadAgentHintsFromSelectionForValidation(os.DirFS("../cli/schema_hints/selection"))
|
|
if err != nil {
|
|
t.Fatalf("LoadAgentHintsFromSelectionForValidation() error = %v", err)
|
|
}
|
|
contractRoot := NewRootCommand()
|
|
if _, err := cli.ApplyEmbeddedManualSchemaHints(contractRoot); err != nil {
|
|
t.Fatalf("ApplyEmbeddedManualSchemaHints() error = %v", err)
|
|
}
|
|
effective, err := cli.BuildEffectiveCommandRegistry(contractRoot)
|
|
if err != nil {
|
|
t.Fatalf("BuildEffectiveCommandRegistry() error = %v", err)
|
|
}
|
|
bound, err := cli.BindEffectiveCommandRegistry(contractRoot, effective)
|
|
if err != nil {
|
|
t.Fatalf("BindEffectiveCommandRegistry() error = %v", err)
|
|
}
|
|
registry, err := cli.AssembleSchemaRegistryFromBound(bound)
|
|
if err != nil {
|
|
t.Fatalf("AssembleSchemaRegistryFromBound() error = %v", err)
|
|
}
|
|
if err := cli.ValidateReviewedDryRunCapabilityDelivery(registry); err != nil {
|
|
t.Fatalf("ValidateReviewedDryRunCapabilityDelivery() error = %v", err)
|
|
}
|
|
plan, err := cli.BuildManualAgentExampleExecutionPlan(bound, registry, hints)
|
|
if err != nil {
|
|
t.Fatalf("BuildManualAgentExampleExecutionPlan() error = %v", err)
|
|
}
|
|
return plan
|
|
}
|
|
|
|
type manualAgentExampleCapture struct {
|
|
Output string
|
|
DryRunChecks int64
|
|
ToolCallAttempts int64
|
|
StdinBytesRead int64
|
|
}
|
|
|
|
type manualAgentExampleFailClosedCaller struct {
|
|
dryRunChecks atomic.Int64
|
|
toolCallAttempts atomic.Int64
|
|
}
|
|
|
|
func (c *manualAgentExampleFailClosedCaller) CallTool(_ context.Context, productID, toolName string, _ map[string]any) (*edition.ToolResult, error) {
|
|
c.toolCallAttempts.Add(1)
|
|
return nil, fmt.Errorf("real ToolCaller invocation blocked during Agent example dry-run: %s/%s", productID, toolName)
|
|
}
|
|
|
|
func (c *manualAgentExampleFailClosedCaller) Format() string { return "json" }
|
|
|
|
func (c *manualAgentExampleFailClosedCaller) DryRun() bool {
|
|
c.dryRunChecks.Add(1)
|
|
return true
|
|
}
|
|
|
|
func (c *manualAgentExampleFailClosedCaller) Fields() string { return "" }
|
|
func (c *manualAgentExampleFailClosedCaller) JQ() string { return "" }
|
|
|
|
func executeManualAgentExampleCapture(t testing.TB, args []string) (manualAgentExampleCapture, error) {
|
|
t.Helper()
|
|
oldArgs := os.Args
|
|
os.Args = append([]string{"dws"}, args...)
|
|
defer func() { os.Args = oldArgs }()
|
|
oldStdin := os.Stdin
|
|
promptInput, err := os.CreateTemp(t.TempDir(), "agent-example-stdin-*.txt")
|
|
if err != nil {
|
|
t.Fatalf("open guarded stdin: %v", err)
|
|
}
|
|
defer promptInput.Close()
|
|
if _, err := promptInput.WriteString("no\n"); err != nil {
|
|
t.Fatalf("seed guarded stdin: %v", err)
|
|
}
|
|
if _, err := promptInput.Seek(0, io.SeekStart); err != nil {
|
|
t.Fatalf("rewind guarded stdin: %v", err)
|
|
}
|
|
os.Stdin = promptInput
|
|
defer func() { os.Stdin = oldStdin }()
|
|
|
|
oldStdout, oldStderr := os.Stdout, os.Stderr
|
|
oldColorOutput, oldColorError := color.Output, color.Error
|
|
captureFile, err := os.CreateTemp(t.TempDir(), "agent-example-output-*.log")
|
|
if err != nil {
|
|
t.Fatalf("open output capture file: %v", err)
|
|
}
|
|
defer captureFile.Close()
|
|
os.Stdout, os.Stderr = captureFile, captureFile
|
|
color.Output, color.Error = captureFile, captureFile
|
|
defer func() {
|
|
os.Stdout, os.Stderr = oldStdout, oldStderr
|
|
color.Output, color.Error = oldColorOutput, oldColorError
|
|
}()
|
|
|
|
root := NewRootCommand()
|
|
originalCaller := helpers.GetCaller()
|
|
auditCaller := &manualAgentExampleFailClosedCaller{}
|
|
helpers.InitDeps(auditCaller)
|
|
defer helpers.InitDeps(originalCaller)
|
|
var output bytes.Buffer
|
|
root.SetOut(&output)
|
|
root.SetErr(&output)
|
|
root.SetArgs(args)
|
|
execErr := root.Execute()
|
|
|
|
os.Stdout, os.Stderr = oldStdout, oldStderr
|
|
color.Output, color.Error = oldColorOutput, oldColorError
|
|
if _, err := captureFile.Seek(0, io.SeekStart); err != nil {
|
|
t.Fatalf("rewind output capture file: %v", err)
|
|
}
|
|
captured, readErr := io.ReadAll(captureFile)
|
|
if readErr != nil {
|
|
t.Fatalf("read output capture file: %v", readErr)
|
|
}
|
|
stdinBytesRead, err := promptInput.Seek(0, io.SeekCurrent)
|
|
if err != nil {
|
|
t.Fatalf("inspect guarded stdin: %v", err)
|
|
}
|
|
return manualAgentExampleCapture{
|
|
Output: output.String() + string(captured),
|
|
DryRunChecks: auditCaller.dryRunChecks.Load(),
|
|
ToolCallAttempts: auditCaller.toolCallAttempts.Load(),
|
|
StdinBytesRead: stdinBytesRead,
|
|
}, execErr
|
|
}
|
|
|
|
type manualAgentExampleFiles struct {
|
|
root string
|
|
markdown string
|
|
json string
|
|
batch string
|
|
binary string
|
|
image string
|
|
}
|
|
|
|
func newManualAgentExampleFiles(t testing.TB, root string) manualAgentExampleFiles {
|
|
t.Helper()
|
|
markdown := filepath.Join(root, "content.md")
|
|
jsonFile := filepath.Join(root, "report.json")
|
|
batch := filepath.Join(root, "styles.json")
|
|
binary := filepath.Join(root, "report.pdf")
|
|
image := filepath.Join(root, "chart.png")
|
|
for path, content := range map[string][]byte{
|
|
markdown: []byte("# Agent dry-run fixture\n\nNo business call is allowed.\n"),
|
|
jsonFile: []byte(`[{"content":"Agent dry-run fixture","sort":"0","key":"fixture","contentType":"markdown","type":"1"}]`),
|
|
batch: []byte(`[{"sheetId":"Sheet1","range":"A1:B2","fontWeight":"bold"}]`),
|
|
binary: []byte("%PDF-1.4\n%%EOF\n"),
|
|
image: {0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'},
|
|
} {
|
|
if err := os.WriteFile(path, content, 0o600); err != nil {
|
|
t.Fatalf("write dry-run fixture %s: %v", path, err)
|
|
}
|
|
}
|
|
return manualAgentExampleFiles{root: root, markdown: markdown, json: jsonFile, batch: batch, binary: binary, image: image}
|
|
}
|
|
|
|
func materializeManualAgentExampleArgv(argv []string, files manualAgentExampleFiles) []string {
|
|
result := append([]string(nil), argv...)
|
|
for index := range result {
|
|
result[index] = manualAgentExamplePlaceholderPattern.ReplaceAllStringFunc(result[index], func(match string) string {
|
|
name := strings.TrimSuffix(strings.TrimPrefix(match, "<"), ">")
|
|
switch strings.ToLower(name) {
|
|
case "basetime", "remindertimestamp", "reminder-time-stamp":
|
|
return "1780000000000"
|
|
case "duedateoffset", "due-date-offset":
|
|
return "0"
|
|
case "reminderrules", "reminder-rules":
|
|
return `[{"remindType":"minute","remindTime":10}]`
|
|
case "filepath", "file-path":
|
|
return files.binary
|
|
case "uuid1,uuid2":
|
|
return "uuid1,uuid2"
|
|
default:
|
|
clean := strings.NewReplacer(",", "_", "-", "_", ".", "_").Replace(name)
|
|
return "test_" + clean
|
|
}
|
|
})
|
|
}
|
|
|
|
for index := 0; index < len(result); index++ {
|
|
name, inline, ok := manualAgentExampleLongFlag(result[index])
|
|
if !ok {
|
|
continue
|
|
}
|
|
valueIndex := index + 1
|
|
value := inline
|
|
if inline == "" && valueIndex < len(result) {
|
|
value = result[valueIndex]
|
|
}
|
|
replacement := ""
|
|
switch name {
|
|
case "file", "file-path":
|
|
if strings.Contains(strings.ToLower(value), "png") {
|
|
replacement = files.image
|
|
} else {
|
|
replacement = files.binary
|
|
}
|
|
case "content-file":
|
|
replacement = files.markdown
|
|
case "contents-file":
|
|
replacement = files.json
|
|
case "batch":
|
|
if strings.HasSuffix(strings.ToLower(value), "styles.json") {
|
|
replacement = files.batch
|
|
}
|
|
case "output":
|
|
if value == "." || value == "" {
|
|
replacement = files.root
|
|
} else {
|
|
replacement = filepath.Join(files.root, filepath.Base(value))
|
|
}
|
|
}
|
|
if replacement == "" {
|
|
continue
|
|
}
|
|
if inline != "" {
|
|
result[index] = "--" + name + "=" + replacement
|
|
} else if valueIndex < len(result) {
|
|
result[valueIndex] = replacement
|
|
index++
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func manualAgentExampleLongFlag(argument string) (name, inline string, ok bool) {
|
|
if !strings.HasPrefix(argument, "--") {
|
|
return "", "", false
|
|
}
|
|
name, inline, _ = strings.Cut(strings.TrimPrefix(argument, "--"), "=")
|
|
return name, inline, name != ""
|
|
}
|
|
|
|
func manualAgentExampleHasFlag(argv []string, target string) bool {
|
|
for _, argument := range argv {
|
|
if argument == "--"+target || strings.HasPrefix(argument, "--"+target+"=") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func manualAgentExampleDryRunObserved(capture manualAgentExampleCapture) bool {
|
|
_, ok := manualAgentExampleDryRunEvidence(capture)
|
|
return ok
|
|
}
|
|
|
|
func manualAgentExampleDryRunEvidence(capture manualAgentExampleCapture) (string, bool) {
|
|
normalized := strings.ToLower(capture.Output)
|
|
if manualAgentExampleDryRunJSONPattern.MatchString(capture.Output) && manualAgentExampleDryRunPlanPattern.MatchString(capture.Output) {
|
|
return cli.DryRunPreviewPlan, true
|
|
}
|
|
if manualAgentExampleDryRunJSONPattern.MatchString(capture.Output) {
|
|
return cli.DryRunPreviewRequest, true
|
|
}
|
|
if strings.Contains(normalized, "[dry-run]") {
|
|
return cli.DryRunPreviewInvocation, true
|
|
}
|
|
if capture.DryRunChecks > 0 && strings.Contains(capture.Output, "操作:") {
|
|
return cli.DryRunPreviewPlan, true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func TestManualAgentExampleDryRunEvidenceAcceptsSharedAndCommandPlans(t *testing.T) {
|
|
if !manualAgentExampleDryRunObserved(manualAgentExampleCapture{Output: "[DRY-RUN] Preview only, not executed:\nTool: calendar_list"}) {
|
|
t.Fatal("dry-run output with a Tool and nil Arguments was not recognized")
|
|
}
|
|
if manualAgentExampleDryRunObserved(manualAgentExampleCapture{Output: "Tool: calendar_list"}) {
|
|
t.Fatal("a Tool line without dry-run evidence must not be accepted")
|
|
}
|
|
for _, falseEvidence := range []string{
|
|
"unknown flag: --dry-run",
|
|
"Run again with --dry-run to preview the operation",
|
|
`{"dry_run":false,"executed":true}`,
|
|
} {
|
|
if manualAgentExampleDryRunObserved(manualAgentExampleCapture{Output: falseEvidence}) {
|
|
t.Errorf("non-evidence text was mistaken for a successful dry-run: %q", falseEvidence)
|
|
}
|
|
}
|
|
operationSummary := "操作: 下载钉盘文件\n文件ID: test"
|
|
if manualAgentExampleDryRunObserved(manualAgentExampleCapture{Output: operationSummary}) {
|
|
t.Fatal("a human-only operation summary without an audited dry-run check must not be accepted")
|
|
}
|
|
if !manualAgentExampleDryRunObserved(manualAgentExampleCapture{Output: operationSummary, DryRunChecks: 1}) {
|
|
t.Fatal("a command plan guarded by the injected caller's dry-run check was not recognized")
|
|
}
|
|
}
|
|
|
|
func TestCrossPlatformCoverageManualAgentExampleDryRunEvidenceClassifiesStructuredPlan(t *testing.T) {
|
|
kind, observed := manualAgentExampleDryRunEvidence(manualAgentExampleCapture{
|
|
Output: `{"dry_run":true,"executed":false,"preview_kind":"plan"}`,
|
|
DryRunChecks: 1,
|
|
})
|
|
if !observed || kind != cli.DryRunPreviewPlan {
|
|
t.Fatalf("structured plan classified as kind=%q observed=%v", kind, observed)
|
|
}
|
|
if manualAgentExampleDryRunObserved(manualAgentExampleCapture{
|
|
Output: `{"dry_run":false,"executed":false,"preview_kind":"plan"}`,
|
|
DryRunChecks: 1,
|
|
}) {
|
|
t.Fatal("non-dry-run structured plan was accepted as evidence")
|
|
}
|
|
}
|
|
|
|
func manualAgentExamplePromptObserved(output string) bool {
|
|
normalized := strings.ToLower(output)
|
|
for _, marker := range []string{
|
|
"confirm ",
|
|
"confirm deletion?",
|
|
"confirm action?",
|
|
"confirm create?",
|
|
"confirm update?",
|
|
"confirm save?",
|
|
"confirm import?",
|
|
"are you sure",
|
|
"operation cancelled",
|
|
"操作已取消",
|
|
} {
|
|
if strings.Contains(normalized, marker) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestManualAgentExamplePromptObservedRejectsInteractiveConfirmation(t *testing.T) {
|
|
for _, prompt := range []string{
|
|
"Confirm deletion? (yes/no):",
|
|
"Confirm action? (yes/no):",
|
|
"Confirm create? (yes/no):",
|
|
"Confirm update? (yes/no):",
|
|
"Confirm save? (yes/no):",
|
|
"Confirm import? (yes/no):",
|
|
"Are you sure you want to continue?",
|
|
"Operation cancelled",
|
|
} {
|
|
if !manualAgentExamplePromptObserved(prompt) {
|
|
t.Errorf("interactive confirmation output was not detected: %q", prompt)
|
|
}
|
|
}
|
|
if manualAgentExamplePromptObserved(`{"dry_run":true,"confirmation":"user_required"}`) {
|
|
t.Fatal("typed safety metadata was mistaken for an interactive prompt")
|
|
}
|
|
}
|
|
|
|
func TestAitableAdvpermDisableDryRunSkipsConfirmationAndToolCall(t *testing.T) {
|
|
setTestHome(t, t.TempDir())
|
|
t.Setenv("DWS_CONFIG_DIR", t.TempDir())
|
|
|
|
args := []string{
|
|
"--dry-run", "--format", "json",
|
|
"aitable", "advperm", "disable",
|
|
"--base-id", "BASE_ID",
|
|
}
|
|
if manualAgentExampleHasFlag(args, "yes") {
|
|
t.Fatal("regression test must not bypass confirmation with --yes")
|
|
}
|
|
capture, err := executeManualAgentExampleCapture(t, args)
|
|
if err != nil {
|
|
t.Fatalf("advperm disable fail-closed dry-run failed: %v\noutput:\n%s", err, capture.Output)
|
|
}
|
|
if capture.StdinBytesRead != 0 || manualAgentExamplePromptObserved(capture.Output) {
|
|
t.Fatalf("advperm disable dry-run entered confirmation (stdin bytes read: %d)\noutput:\n%s", capture.StdinBytesRead, capture.Output)
|
|
}
|
|
if capture.ToolCallAttempts != 0 {
|
|
t.Fatalf("advperm disable dry-run attempted %d real ToolCaller invocation(s)\noutput:\n%s", capture.ToolCallAttempts, capture.Output)
|
|
}
|
|
if !manualAgentExampleDryRunObserved(capture) {
|
|
t.Fatalf("advperm disable returned no audited dry-run evidence (caller dry-run checks: %d)\noutput:\n%s", capture.DryRunChecks, capture.Output)
|
|
}
|
|
}
|
|
|
|
func TestManualAgentExampleFailClosedCallerRecordsToolCalls(t *testing.T) {
|
|
caller := &manualAgentExampleFailClosedCaller{}
|
|
if !caller.DryRun() {
|
|
t.Fatal("fail-closed caller must advertise dry-run mode")
|
|
}
|
|
if _, err := caller.CallTool(context.Background(), "calendar", "list_events", nil); err == nil {
|
|
t.Fatal("fail-closed caller accepted a ToolCaller invocation")
|
|
}
|
|
if got := caller.dryRunChecks.Load(); got != 1 {
|
|
t.Fatalf("DryRun() checks = %d, want 1", got)
|
|
}
|
|
if got := caller.toolCallAttempts.Load(); got != 1 {
|
|
t.Fatalf("CallTool() attempts = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestManualAgentExampleChatGroupMuteMemberUsesCommandDryRunPreview(t *testing.T) {
|
|
sandboxRoot := t.TempDir()
|
|
configDir := filepath.Join(sandboxRoot, "config")
|
|
if err := os.MkdirAll(configDir, 0o700); err != nil {
|
|
t.Fatalf("create isolated config directory: %v", err)
|
|
}
|
|
setTestHome(t, sandboxRoot)
|
|
t.Setenv("DWS_CONFIG_DIR", configDir)
|
|
|
|
capture, err := executeManualAgentExampleCapture(t, []string{
|
|
"--dry-run",
|
|
"chat", "group-mute-member",
|
|
"--group", "test_openConversationId",
|
|
"--users", "userId1,userId2",
|
|
"--mute-time", "3600000",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("group-mute-member dry-run failed: %v\noutput:\n%s", err, capture.Output)
|
|
}
|
|
if capture.ToolCallAttempts != 0 {
|
|
t.Fatalf("group-mute-member dry-run attempted %d ToolCaller invocation(s)\noutput:\n%s", capture.ToolCallAttempts, capture.Output)
|
|
}
|
|
if capture.DryRunChecks == 0 {
|
|
t.Fatalf("group-mute-member did not enter its audited command dry-run path\noutput:\n%s", capture.Output)
|
|
}
|
|
if capture.StdinBytesRead != 0 || manualAgentExamplePromptObserved(capture.Output) {
|
|
t.Fatalf("group-mute-member dry-run entered an interactive prompt (stdin bytes read: %d)\noutput:\n%s", capture.StdinBytesRead, capture.Output)
|
|
}
|
|
if !manualAgentExampleDryRunObserved(capture) {
|
|
t.Fatalf("group-mute-member returned no audited dry-run evidence\noutput:\n%s", capture.Output)
|
|
}
|
|
for _, expected := range []string{`"uids"`, `"userId1"`, `"userId2"`} {
|
|
if !strings.Contains(capture.Output, expected) {
|
|
t.Fatalf("group-mute-member command preview missing %s\noutput:\n%s", expected, capture.Output)
|
|
}
|
|
}
|
|
if strings.Contains(capture.Output, `"openDingTalkIds"`) {
|
|
t.Fatalf("group-mute-member dry-run unexpectedly resolved user IDs remotely\noutput:\n%s", capture.Output)
|
|
}
|
|
}
|