feat: declarative LeafSpec command framework + schema generation/consumption separation
== LeafSpec command framework (internal/helpers/leaf.go) ==
Declarative command construction: LeafSpec/LeafFlag/NewLeafCommand with
Call (pluggable dispatch), LeafInt, PostMount, Trim, Validate. Collapses
per-command hand-written required validation, alias/env fallback, value
transform, and toolArgs assembly into one declarative path.
== devapp migration (28/31 commands) ==
All MCP-direct devapp leaf commands migrated to LeafSpec. Factories
(devAppCall/devAppCallCursor/devAppMeta) fold 33 repeated closures.
fakeDevAppRunner asserts toolArgs for every migrated command. 4 complex
commands (delete/robot submit/result/config) kept hand-written.
== Schema generation/consumption separation ==
- gen.go: isolated //go:generate pragmas from business code.
- command_meta.go: ResolveMeta(cliPath) -> CommandMeta{Identity,Safety,Selection}.
- command_safety.go: SafetyForCLIPath + RenderSafetyAnnotation; safety metadata
flows from embedded catalog into --help output.
- calendar.go HelpFunc fix: delegates to root HelpFunc at help-time.
- schema_catalog_structure.go: closed catalog structure validation gate.
== Registry + catalog per-product sharding ==
schema_command_registry and schema_catalog split into per-product shards,
eliminating concurrent-PR merge conflicts on these files.
== MCP metadata refresh tool ==
cmd/fetch_mcp_metadata: iterates 26 MCP server endpoints, merges with previous
data for cross-server interface_ref. make fetch-mcp-metadata target.
== AGENTS.md ==
Documents the generation/consumption split.
Verified: make policy exit 0, drift zero, all tests pass.
This commit is contained in:
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
# scripts/dev/fetch_mcp_metadata.sh
|
||||
#
|
||||
# Refreshes internal/cli/schema_mcp_metadata.json from the live MCP server.
|
||||
# DWS's equivalent of lark-cli's scripts/fetch_meta.py.
|
||||
#
|
||||
# Prerequisites:
|
||||
# dws auth login (valid access token required)
|
||||
#
|
||||
# Usage:
|
||||
# make fetch-mcp-metadata
|
||||
# # or directly:
|
||||
# scripts/dev/fetch_mcp_metadata.sh
|
||||
#
|
||||
# The script tries to extract the access token from the DWS auth store.
|
||||
# If that fails (e.g. keychain-based storage), set DWS_ACCESS_TOKEN manually:
|
||||
# export DWS_ACCESS_TOKEN=<your-token>
|
||||
# scripts/dev/fetch_mcp_metadata.sh
|
||||
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# --- 1. Check auth ---
|
||||
AUTH_JSON=$(dws auth status --format json 2>/dev/null || echo '{}')
|
||||
if ! echo "$AUTH_JSON" | grep -q '"authenticated": *true'; then
|
||||
echo "Error: not authenticated. Run 'dws auth login' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ Authenticated." >&2
|
||||
|
||||
# --- 2. Try to extract access token ---
|
||||
CONFIG_DIR="${DWS_CONFIG_DIR:-$HOME/.dws}"
|
||||
TOKEN_FILE="$CONFIG_DIR/token"
|
||||
|
||||
if [ -z "${DWS_ACCESS_TOKEN:-}" ] && [ -f "$TOKEN_FILE" ]; then
|
||||
# Try file-based token store (plaintext JSON)
|
||||
DWS_ACCESS_TOKEN=$(python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(open('$TOKEN_FILE'))
|
||||
t = d.get('accessToken') or d.get('access_token') or ''
|
||||
print(t)
|
||||
except:
|
||||
pass
|
||||
" 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
if [ -z "${DWS_ACCESS_TOKEN:-}" ]; then
|
||||
echo "Warning: could not auto-extract access token." >&2
|
||||
echo " Token file: $TOKEN_FILE" >&2
|
||||
echo " If using keychain, set DWS_ACCESS_TOKEN manually:" >&2
|
||||
echo " export DWS_ACCESS_TOKEN=<your-token>" >&2
|
||||
echo " Continuing without auth (may fail)..." >&2
|
||||
else
|
||||
echo "✓ Got access token (${#DWS_ACCESS_TOKEN} chars)." >&2
|
||||
fi
|
||||
|
||||
export DWS_ACCESS_TOKEN
|
||||
|
||||
# --- 3. Run the Go tool ---
|
||||
go run ./cmd/fetch_mcp_metadata \
|
||||
-output internal/cli/schema_mcp_metadata.json \
|
||||
2>&1
|
||||
|
||||
echo "" >&2
|
||||
echo "Done. Review changes with: git diff internal/cli/schema_mcp_metadata.json" >&2
|
||||
echo "Then commit if the refreshed metadata looks correct." >&2
|
||||
@@ -2,7 +2,11 @@
|
||||
set -eu
|
||||
|
||||
# Check command surface for drift.
|
||||
# Stub placeholder — extend with actual surface comparison logic as needed.
|
||||
#
|
||||
# 硬门禁:校验内嵌 schema_catalog.json 的封闭结构(字段白名单 + 枚举值 +
|
||||
# 交叉一致性),由 internal/cli/schema_catalog_structure.go 的
|
||||
# TestEmbeddedSchemaCatalogStructure 实现。该门禁确保任何写入 catalog 的代码
|
||||
# 路径都不会产出结构非法的工具条目。
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
BIN="${DWS_BIN:-$ROOT/dws}"
|
||||
@@ -19,6 +23,12 @@ if [ ! -x "$BIN" ]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# 硬门禁:内嵌 schema_catalog.json 封闭结构校验。
|
||||
if ! go test ./internal/cli -run '^TestEmbeddedSchemaCatalogStructure$' -count=1; then
|
||||
printf 'embedded schema catalog structure check: FAILED\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# These utility commands are stable across the current open-source CLI shape.
|
||||
EXPECTED_COMMANDS="auth cache completion version"
|
||||
missing=0
|
||||
|
||||
@@ -21,8 +21,8 @@ go build -a -o "$catalog_generator" ./internal/generator/cmd_schema_catalog
|
||||
|
||||
# CommandRegistry is a reviewed input, never a generated artifact. Keep an
|
||||
# independent byte-for-byte guard around the ordinary downstream generators.
|
||||
registry_guard="$tmp/schema_command_registry.json"
|
||||
cp internal/cli/schema_command_registry.json "$registry_guard"
|
||||
registry_guard="$tmp/schema_command_registry"
|
||||
cp -R internal/cli/schema_command_registry "$registry_guard"
|
||||
# Also guard human-authored metadata + selection hint trees.
|
||||
metadata_guard="$tmp/metadata-hints"
|
||||
selection_guard="$tmp/selection-hints"
|
||||
@@ -31,12 +31,12 @@ cp -R internal/cli/schema_hints/selection "$selection_guard"
|
||||
|
||||
metadata_tmp="$tmp/metadata"
|
||||
audit_tmp="$tmp/audit.json"
|
||||
catalog_tmp="$tmp/catalog.json"
|
||||
catalog_tmp_second="$tmp/catalog-second.json"
|
||||
catalog_tmp="$tmp/schema_catalog"
|
||||
catalog_tmp_second="$tmp/schema_catalog-second"
|
||||
|
||||
"$metadata_generator" \
|
||||
-root . \
|
||||
-registry internal/cli/schema_command_registry.json \
|
||||
-registry internal/cli/schema_command_registry \
|
||||
-output-dir "$metadata_tmp" \
|
||||
-audit-output "$audit_tmp"
|
||||
|
||||
@@ -62,8 +62,8 @@ fi
|
||||
-root . \
|
||||
-output "$catalog_tmp_second"
|
||||
|
||||
if ! cmp -s internal/cli/schema_command_registry.json "$registry_guard"; then
|
||||
printf '%s\n' 'generation modified reviewed input internal/cli/schema_command_registry.json' >&2
|
||||
if ! diff -qr internal/cli/schema_command_registry "$registry_guard" >/dev/null; then
|
||||
printf '%s\n' 'generation modified reviewed input internal/cli/schema_command_registry/' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -77,16 +77,16 @@ if ! diff -qr internal/cli/schema_hints/selection "$selection_guard" >/dev/null;
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cmp -s "$catalog_tmp" "$catalog_tmp_second"; then
|
||||
if ! diff -qr "$catalog_tmp" "$catalog_tmp_second" >/dev/null; then
|
||||
printf '%s\n' 'generated drift: consecutive Catalog generations are not byte-identical' >&2
|
||||
diff -u "$catalog_tmp" "$catalog_tmp_second" || true
|
||||
diff -ru "$catalog_tmp" "$catalog_tmp_second" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cmp -s internal/cli/schema_catalog.json "$catalog_tmp"; then
|
||||
printf '%s\n' 'generated drift: internal/cli/schema_catalog.json is stale' >&2
|
||||
if ! diff -qr internal/cli/schema_catalog "$catalog_tmp" >/dev/null; then
|
||||
printf '%s\n' 'generated drift: internal/cli/schema_catalog is stale' >&2
|
||||
printf '%s\n' 'run: make generate-schema' >&2
|
||||
diff -u internal/cli/schema_catalog.json "$catalog_tmp" || true
|
||||
diff -ru internal/cli/schema_catalog "$catalog_tmp" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
metadata_dir="internal/cli/schema_hints/metadata"
|
||||
catalog="internal/cli/schema_catalog.json"
|
||||
|
||||
if [ ! -d "$metadata_dir" ]; then
|
||||
printf '%s\n' "missing agent metadata directory: $metadata_dir" >&2
|
||||
@@ -18,6 +17,11 @@ fi
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
# The release Catalog is committed as a per-product split; reassemble it into
|
||||
# the single-document shape these jq queries consume.
|
||||
catalog="$tmp/catalog-combined.json"
|
||||
scripts/policy/with-catalog.sh >"$catalog"
|
||||
|
||||
jq -r '
|
||||
.tools
|
||||
| to_entries[]
|
||||
|
||||
@@ -2,23 +2,24 @@
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
BIN="${DWS_BIN:-$ROOT/dws}"
|
||||
cd "$ROOT"
|
||||
. "$ROOT/scripts/policy/policy-runtime.sh"
|
||||
policy_prepare_runtime "$ROOT"
|
||||
|
||||
BIN="${DWS_BIN:-$ROOT/dws}"
|
||||
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
# The release Catalog is committed as a per-product split; reassemble it into
|
||||
# the single-document shape this check consumes.
|
||||
catalog_combined="$tmp/catalog-combined.json"
|
||||
scripts/policy/with-catalog.sh >"$catalog_combined"
|
||||
|
||||
if [ ! -x "$BIN" ]; then
|
||||
printf 'error: dws binary not found at %s (run make build first)\n' "$BIN" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
tmp="$(mktemp -d)"
|
||||
exec_tmp="$(policy_runtime_mktemp_dir dws-schema-binary)"
|
||||
smoke_generator="$exec_tmp/schema-registry-smoke"
|
||||
trap 'rm -rf "$tmp" "$exec_tmp"' EXIT HUP INT TERM
|
||||
|
||||
go build -o "$smoke_generator" ./internal/generator/cmd_schema_registry_smoke
|
||||
"$smoke_generator" >"$tmp/smoke-vector.json"
|
||||
go run ./internal/generator/cmd_schema_registry_smoke >"$tmp/smoke-vector.json"
|
||||
|
||||
"$BIN" schema list --format json >"$tmp/list.json"
|
||||
"$BIN" schema --all --format json >"$tmp/all.json"
|
||||
@@ -43,7 +44,7 @@ fi
|
||||
|
||||
if ! jq -e -n \
|
||||
--slurpfile vector "$tmp/smoke-vector.json" \
|
||||
--slurpfile catalog internal/cli/schema_catalog.json \
|
||||
--slurpfile catalog "$catalog_combined" \
|
||||
--slurpfile list "$tmp/list.json" \
|
||||
--slurpfile all "$tmp/all.json" '
|
||||
$vector[0] as $vector |
|
||||
@@ -103,7 +104,7 @@ jq -S '
|
||||
(if $catalog | has("agent_metadata") then {agent_metadata: $catalog.agent_metadata} else {} end) +
|
||||
{catalog_hash: $snapshot.source_hash} +
|
||||
(if (($snapshot.surface_hash // "") | nonempty) then {surface_hash: $snapshot.surface_hash} else {} end)
|
||||
' internal/cli/schema_catalog.json >"$tmp/expected-list.json"
|
||||
' "$catalog_combined" >"$tmp/expected-list.json"
|
||||
jq -S . "$tmp/list.json" >"$tmp/actual-list.json"
|
||||
if ! cmp -s "$tmp/expected-list.json" "$tmp/actual-list.json"; then
|
||||
printf '%s\n' 'built dws schema list differs from the typed Catalog overview projection' >&2
|
||||
|
||||
@@ -7,6 +7,13 @@ cd "$ROOT"
|
||||
. "$ROOT/scripts/policy/policy-runtime.sh"
|
||||
policy_prepare_runtime "$ROOT"
|
||||
|
||||
# The release Catalog is committed as a per-product split; reassemble it into
|
||||
# the single-document shape (version + surface_hash + source_hash + catalog +
|
||||
# tools) that the jq queries below consume.
|
||||
catalog="$(mktemp)"
|
||||
trap 'rm -f "$catalog"' EXIT HUP INT TERM
|
||||
scripts/policy/with-catalog.sh >"$catalog"
|
||||
|
||||
if [ -e internal/cli/schema_native_contracts.go ] ||
|
||||
[ -e internal/cli/schema_native_contracts_generated.go ] ||
|
||||
policy_search_go 'ApplyNativeRuntimeSchemaContracts|nativeRuntimeSchemaContracts|runtimeSchemaIdentityCandidate' internal/cli; then
|
||||
@@ -17,9 +24,9 @@ fi
|
||||
# The Go delivery gates compare final content against the reviewed
|
||||
# CommandRegistry. This shell check intentionally treats Catalog as output and
|
||||
# does not decode the registry into a competing identity model.
|
||||
registry_count="$(jq -r '.tools | length' internal/cli/schema_catalog.json)"
|
||||
registry_count="$(jq -r '.tools | length' "$catalog")"
|
||||
catalog_count="$registry_count"
|
||||
catalog_product_count="$(jq -r '.catalog.count' internal/cli/schema_catalog.json)"
|
||||
catalog_product_count="$(jq -r '.catalog.count' "$catalog")"
|
||||
mcp_snapshot_registry_count="$(jq -r '.coverage.surface_tools' internal/cli/schema_mcp_metadata.json)"
|
||||
agent_registry_count="$(jq -r '.coverage.surface_tools' internal/cli/schema_agent_metadata/index.json)"
|
||||
agent_product_count="$(jq -r '.coverage.products_with_metadata' internal/cli/schema_agent_metadata/index.json)"
|
||||
@@ -101,7 +108,7 @@ if ! jq -e '
|
||||
exit 1
|
||||
fi
|
||||
|
||||
catalog_registry_hash="$(jq -r '.surface_hash' internal/cli/schema_catalog.json)"
|
||||
catalog_registry_hash="$(jq -r '.surface_hash' "$catalog")"
|
||||
agent_registry_hash="$(jq -r '.surface_hash' internal/cli/schema_agent_metadata/index.json)"
|
||||
if [ "$catalog_registry_hash" != "$agent_registry_hash" ]; then
|
||||
printf 'schema CommandRegistry hashes disagree: catalog=%s agent=%s\n' \
|
||||
@@ -138,7 +145,7 @@ if ! jq -e --arg registry_count "$registry_count" '
|
||||
else false end) and
|
||||
(((.agent_source_refs // []) | map(test("schema_hints/selection/")) | any))
|
||||
)
|
||||
' internal/cli/schema_catalog.json >/dev/null; then
|
||||
' "$catalog" >/dev/null; then
|
||||
printf '%s\n' 'schema tools must have complete Agent summary/effect/safety metadata' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -148,7 +155,7 @@ if ! jq -e 'all(.tools[];
|
||||
elif .interface_mode == "mcp" then .interface_ref != null
|
||||
else .interface_ref == null
|
||||
end
|
||||
)' internal/cli/schema_catalog.json >/dev/null; then
|
||||
)' "$catalog" >/dev/null; then
|
||||
printf '%s
|
||||
' 'schema interface disposition is inconsistent with interface_ref presence' >&2
|
||||
exit 1
|
||||
@@ -174,7 +181,7 @@ if ! jq -e '
|
||||
"rpc_name": "send_personal_message"
|
||||
} and
|
||||
(.tools | has("chat.upload_conversation_file") | not)
|
||||
' internal/cli/schema_catalog.json >/dev/null; then
|
||||
' "$catalog" >/dev/null; then
|
||||
printf '%s\n' 'chat send/reply schema identities are inconsistent' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -188,7 +195,7 @@ if ! jq -e '
|
||||
all((.constraints.require_one_of // [])[]; length > 1 and all(.[]; IN($names[]))) and
|
||||
all((.constraints.require_together // [])[]; length > 1 and all(.[]; IN($names[])))
|
||||
)
|
||||
' internal/cli/schema_catalog.json >/dev/null; then
|
||||
' "$catalog" >/dev/null; then
|
||||
printf '%s\n' 'schema command constraints are incomplete or reference unknown parameters' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -226,7 +233,7 @@ if ! jq -e --slurpfile bindings internal/cli/schema_parameter_bindings.json '
|
||||
. as $binding |
|
||||
$catalog.tools[$binding.tool].parameters[$binding.flag].property == $binding.property
|
||||
)
|
||||
' internal/cli/schema_catalog.json >/dev/null; then
|
||||
' "$catalog" >/dev/null; then
|
||||
printf 'schema parameter bindings are incomplete or differ from generated catalog: count=%s\n' "$binding_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -236,13 +243,13 @@ if ! jq -e '
|
||||
has("endpoint") or has("auth_headers") or has("authorization") or
|
||||
has("access_token") or has("client_secret")
|
||||
)] | length == 0
|
||||
' internal/cli/schema_catalog.json >/dev/null; then
|
||||
' "$catalog" >/dev/null; then
|
||||
printf '%s\n' 'schema catalog contains runtime endpoint or credential fields' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if policy_search_paths 'mcp-gw\.dingtalk\.com|mcp\.dingtalk\.com/server|Authorization[^[:alnum:]]*:|Bearer [A-Za-z0-9]|access[_-]?token|client[_-]?secret' \
|
||||
internal/cli/schema_catalog.json \
|
||||
"$catalog" \
|
||||
internal/cli/schema_mcp_metadata.json \
|
||||
internal/cli/schema_mcp_service_review.json \
|
||||
internal/cli/schema_agent_metadata \
|
||||
|
||||
@@ -30,17 +30,16 @@ fi
|
||||
if [ ! -f internal/cli/schema_command_registry.schema.json ] ||
|
||||
! jq -e '
|
||||
."$schema" == "./schema_command_registry.schema.json" and
|
||||
.version == 1 and
|
||||
(.products | type) == "array" and
|
||||
(.products | length) > 0
|
||||
' internal/cli/schema_command_registry.json >/dev/null; then
|
||||
printf '%s\n' 'reviewed CommandRegistry is missing its local JSON Schema contract' >&2
|
||||
.version == 1
|
||||
' internal/cli/schema_command_registry/registry.json >/dev/null ||
|
||||
! ls internal/cli/schema_command_registry/products/*.json >/dev/null 2>&1; then
|
||||
printf '%s\n' 'reviewed CommandRegistry is missing its local JSON Schema contract or product shards' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The registry argument is validation-only and all go:generate outputs must be
|
||||
# downstream assets. Reject any directive that targets the reviewed input.
|
||||
if ! grep -Eq '^//go:generate .*cmd_schema_agent_metadata .* -registry internal/cli/schema_command_registry\.json ' internal/cli/schema_agent_metadata.go; then
|
||||
if ! grep -Eq '^//go:generate .*cmd_schema_agent_metadata .* -registry internal/cli/schema_command_registry ' internal/cli/gen.go; then
|
||||
printf '%s\n' 'go generate must validate the reviewed CommandRegistry explicitly' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -110,4 +109,4 @@ go test ./internal/app \
|
||||
-count=1
|
||||
|
||||
printf 'schema CommandRegistry check: ok (%s reviewed commands)\n' \
|
||||
"$(jq '[.products[].tools[]] | length' internal/cli/schema_command_registry.json)"
|
||||
"$(cat internal/cli/schema_command_registry/products/*.json | jq -s '[.[].tools[]] | length')"
|
||||
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Reassemble the split release Catalog shards back into the single JSON
|
||||
# document shape (version + surface_hash + source_hash + catalog + tools) that
|
||||
# the policy jq queries consume.
|
||||
#
|
||||
# The Catalog is a committed global file (schema_catalog/catalog.json); each
|
||||
# product's leaf ToolSpecs live in their own shard (schema_catalog/tools/*.json)
|
||||
# so concurrent feature PRs only rewrite one shard. This helper merges the
|
||||
# shards back into one document so existing jq queries keep working unchanged.
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
dir="$ROOT/internal/cli/schema_catalog"
|
||||
|
||||
jq -s '
|
||||
.[0] as $envelope |
|
||||
($envelope.surface_hash // null) as $surface |
|
||||
{
|
||||
version: $envelope.version,
|
||||
source_hash: $envelope.source_hash,
|
||||
catalog: $envelope.catalog,
|
||||
tools: (reduce .[1:][] as $shard ({}; . + ($shard.tools // {})))
|
||||
} +
|
||||
(if $surface then {surface_hash: $surface} else {} end)
|
||||
' "$dir/catalog.json" "$dir"/tools/*.json
|
||||
Reference in New Issue
Block a user