From 9460437c9af94efb8a60624a8f2c407f06ea98a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=89=E6=BE=9C?= Date: Fri, 5 Jun 2026 17:08:55 +0800 Subject: [PATCH] test: add ambiguous app routing cases --- docs/devapp-yulan-command-routing.md | 2 +- skills/mono/references/intent-guide.md | 10 +++++++- test/run_skill_tests.py | 8 +++++- test/skill_tests.md | 19 ++++++++++++++ test/skill_tests_results.md | 34 ++++++++++++++++++++++++-- 5 files changed, 68 insertions(+), 5 deletions(-) diff --git a/docs/devapp-yulan-command-routing.md b/docs/devapp-yulan-command-routing.md index 32ef63b4..8fe53a59 100644 --- a/docs/devapp-yulan-command-routing.md +++ b/docs/devapp-yulan-command-routing.md @@ -20,7 +20,7 @@ external release condition until the rollout checklist in section 13 passes. | Mono skill routing | Done in repo | `skills/mono/SKILL.md` routes Open Platform app intents to `devapp`. | | Mono product reference | Done in repo | `skills/mono/references/products/devapp.md` summarizes app CRUD, permissions, credentials, events, versions, and troubleshooting. | | Multi product skill | Done in repo | `skills/multi/dingtalk-devapp/` defines the standalone `dingtalk-devapp` skill and reference file. | -| Agent intent test cases | Done in repo | `test/skill_tests.md` includes 21 devapp cases plus devdoc/workbench counterexamples for the overloaded word `应用`; `test/skill_tests_results.md` reports those cases passing. | +| Agent intent test cases | Done in repo | `test/skill_tests.md` includes 21 devapp cases, devdoc/workbench counterexamples, and routing clarification cases for the overloaded word `应用`; `test/skill_tests_results.md` reports those cases passing. | | Skill test parser | Done in repo | `test/run_skill_tests.py` now preserves the current `#### dws ...` command section for each case. | | Static/e2e skill verifier handling | Done in repo | `test/skill_static` and `test/skill_e2e` skip `dws devapp ...` as design-only until the MCP product is published. | | Sensitive config guard | Done in repo | `.gitignore` excludes local `docs/mcp/serviceconfig-pre*` files so personal MCP gateway keys are not staged. | diff --git a/skills/mono/references/intent-guide.md b/skills/mono/references/intent-guide.md index 30a42ae2..35b5c092 100644 --- a/skills/mono/references/intent-guide.md +++ b/skills/mono/references/intent-guide.md @@ -11,6 +11,9 @@ | "查开放平台应用 DemoApp 的 appKey" | 查询开发者后台企业内部应用 | `devapp` | `devdoc` | 目标是应用实例和凭证/agentId，不是接口文档 | | "给应用申请 qyapi_robot_sendmsg 权限" | 管理开放平台应用权限 | `devapp permission` | `devdoc` | 权限点申请属于应用配置变更 | | "查工作台应用 appXYZ 的详情" | 查询钉钉工作台应用 | `workbench app` | `devapp` | `appXYZ/app001` 这类工作台 app id 不是开放平台 `unifiedAppId` | +| "帮我创建一个应用" | 应用类型不明确 | 先追问 | `devapp create` | 缺少开放平台/工作台/AI/MCP 等上下文，不能默认创建企业内部应用 | +| "做一个 AI 应用" | AI 应用/智能体需求不明确 | 先追问或转对应能力 | `devapp create` | 没有开发者后台企业内部应用标识，不能默认走 OpenDev app CRUD | +| "创建 MCP 服务并配置 HSF tool" | MCP 平台工具配置 | OpenDev MCP 平台流程 | `devapp create` | 这是 connector/tool 配置，不是创建开放平台企业内部应用 | | "帮我建一个项目跟踪表" | 创建数据表格 | `aitable` | `todo` | 涉及结构化数据/行列操作，不是个人待办 | | "帮我记一下明天要做的事" | 创建个人待办 | `todo` | `aitable` | 个人待办提醒，非数据表 | | "帮我建一个明天下午的日程" | 日历日程 | `calendar` | — | 日历日程管理（可含参与者/会议室）| @@ -61,7 +64,12 @@ - "工作台应用/钉钉工作台上的应用" - "应用 app001/appXYZ 的详情" 且没有开放平台上下文 -**判断关键**：是否在管理一个开发者后台企业内部应用实例？是 → `devapp`。是否在查 API 文档？是 → `devdoc`。是否在查钉钉工作台入口应用？是 → `workbench app`。 +**先追问的场景**： +- "创建应用/做个应用/查应用" 但没有开放平台、工作台、文档、AI、MCP 等上下文 +- "AI 应用/智能体" 但没有说明是开发者后台企业内部应用还是其他产品能力 +- "MCP 服务/connector/HSF tool" 属于平台配置流程，不要转成 `devapp create` + +**判断关键**：是否在管理一个开发者后台企业内部应用实例？是 → `devapp`。是否在查 API 文档？是 → `devdoc`。是否在查钉钉工作台入口应用？是 → `workbench app`。仍不明确 → 追问，不调用写命令。 --- diff --git a/test/run_skill_tests.py b/test/run_skill_tests.py index b0e37389..57b99629 100644 --- a/test/run_skill_tests.py +++ b/test/run_skill_tests.py @@ -261,7 +261,7 @@ def run_test(test_case: TestCase) -> dict: 'flags_pass': False, 'overall_pass': False, 'details': [], - 'skill_reference': f'references/products/{get_reference_file(test_case.product)}' + 'skill_reference': get_reference_path(test_case.product) } # Extract expected command path @@ -337,6 +337,12 @@ def get_reference_file(product: str) -> str: } return product_to_file.get(product, f'{product}.md') +def get_reference_path(product: str) -> str: + """Get the display path for the skill reference used by a product test.""" + if product == 'routing': + return 'references/intent-guide.md' + return f'references/products/{get_reference_file(product)}' + def generate_report(results: list[dict]) -> str: """Generate markdown report from test results""" report = [] diff --git a/test/skill_tests.md b/test/skill_tests.md index 6d4f4735..5ea25e3c 100644 --- a/test/skill_tests.md +++ b/test/skill_tests.md @@ -48,6 +48,7 @@ Agent 安装 dws skill 后，仅依据 skill 提供的参考文档，将自然 | `devapp` | `references/products/devapp.md` | 14 | 21 | | `ding` | `references/products/ding.md` | 2 | 5 | | `report` | `references/products/report.md` | 6 | 26 | +| `routing` | `references/intent-guide.md` | 1 | 3 | | `todo` | `references/products/todo.md` | 6 | 30 | | `workbench` | `references/products/workbench.md` | 2 | 6 | @@ -1180,6 +1181,24 @@ Agent 安装 dws skill 后，仅依据 skill 提供的参考文档，将自然 --- +### routing（3 条） + +#### `dws --help` + +**routing_routing_clarify_001** `[ASK_USER]` +- Prompt: 帮我创建一个应用 +- Expected: `dws --help` + +**routing_routing_clarify_002** `[ASK_USER]` +- Prompt: 做一个 AI 应用 +- Expected: `dws --help` + +**routing_routing_clarify_003** `[ASK_USER]` +- Prompt: 创建 MCP 服务并配置 HSF tool +- Expected: `dws --help` + +--- + ### todo（30 条） #### `dws todo task create` diff --git a/test/skill_tests_results.md b/test/skill_tests_results.md index ab429180..37e0ff85 100644 --- a/test/skill_tests_results.md +++ b/test/skill_tests_results.md @@ -2,8 +2,8 @@ ## Summary -- **Total Test Cases**: 227 -- **Passed**: 227 +- **Total Test Cases**: 230 +- **Passed**: 230 - **Failed**: 0 - **Pass Rate**: 100.0% @@ -20,6 +20,7 @@ | devdoc | 7 | 7 | 0 | 100.0% | | ding | 5 | 5 | 0 | 100.0% | | report | 26 | 26 | 0 | 100.0% | +| routing | 3 | 3 | 0 | 100.0% | | todo | 30 | 30 | 0 | 100.0% | | workbench | 6 | 6 | 0 | 100.0% | @@ -1573,6 +1574,35 @@ - Command path: PASS (report template list) - Flags: N/A (no flags specified) +### routing + +**routing_routing_clarify_001** ✅ PASS + +- Prompt: 帮我创建一个应用 +- Expected: `dws --help` +- Skill Reference: references/intent-guide.md +- Command path: PASS (--help) +- Flags: N/A (no flags specified) +- Note: [ASK_USER] case - only command path validated + +**routing_routing_clarify_002** ✅ PASS + +- Prompt: 做一个 AI 应用 +- Expected: `dws --help` +- Skill Reference: references/intent-guide.md +- Command path: PASS (--help) +- Flags: N/A (no flags specified) +- Note: [ASK_USER] case - only command path validated + +**routing_routing_clarify_003** ✅ PASS + +- Prompt: 创建 MCP 服务并配置 HSF tool +- Expected: `dws --help` +- Skill Reference: references/intent-guide.md +- Command path: PASS (--help) +- Flags: N/A (no flags specified) +- Note: [ASK_USER] case - only command path validated + ### todo **todo_todo_task_create_001** ✅ PASS