玉澜
2dc3147cda
- drop the algorithm/gate source SHA256 freeze: every ranking change required a manual manifest hash sync, coupling code evolution to the manifest - drop the --require-sealed release gate (sealed qrels hash, dual signatures, coverage minima, independent-result thresholds) and its tool-search-independent-data-gate Makefile target - behavior regression protection stays with the Go evaluation sentinels that run the real assembled catalog
54 lines
2.2 KiB
JSON
54 lines
2.2 KiB
JSON
{
|
|
"version": "tool-search-eval-manifest.v1",
|
|
"created_at": "2026-08-12T00:00:00+08:00",
|
|
"base_repository_commit": "5fed80fc0f1aaa4ee9123b0acf3041d0d9b45bc9",
|
|
"owners": {
|
|
"retrieval": "DWS Tool Search maintainers",
|
|
"independent_evaluation": "DWS Agent Evaluation maintainers",
|
|
"separation_rule": "The independent evaluation signer must not be an author of Catalog selection metadata used by the evaluated cases."
|
|
},
|
|
"algorithm_freeze": {
|
|
"state": "development_candidate_recorded_before_independent_dev_test_is_sealed",
|
|
"diagnostic_proxy_control": "bm25_unfielded",
|
|
"go_runtime_default": "fielded_bm25_ensemble",
|
|
"shadow_candidates": [
|
|
"fielded_bm25_action_v1",
|
|
"fielded_tfidf_cosine",
|
|
"bm25plus"
|
|
],
|
|
"changes_forbidden_after_independent_dev_test_is_sealed": [
|
|
"tokenizer",
|
|
"field_projection",
|
|
"field_weights",
|
|
"bm25_k1_b_delta"
|
|
]
|
|
},
|
|
"proxy_v1": {
|
|
"state": "sealed_diagnostic_only",
|
|
"catalog_generation": "go_declarations_to_runtime_assembly",
|
|
"catalog_build_output": ".worktrees/policy-tmp/tool-search-schema-catalog",
|
|
"catalog_source_hash": "sha256:02c633c075f4af915ea097d383bdcbc25fe549c172e2d3336152e30cf2906e80",
|
|
"catalog_surface_hash": "sha256:7f9839f1ff428b52f1d032d2fa6493815c3755650b7d9fa653839e7d589482e5",
|
|
"tool_count": 1098,
|
|
"workflow_path": "scripts/testdata/tool_search_workflows.json",
|
|
"evaluator_path": "scripts/dev/eval_tool_search_ranking.py",
|
|
"intent_cases": 1123,
|
|
"intent_excluded_over_budget": 10,
|
|
"chinese_only_cases": 402,
|
|
"mixed_chinese_ascii_cases": 721,
|
|
"english_cases": 0,
|
|
"workflow_cases": 10,
|
|
"release_gate": false,
|
|
"limitations": [
|
|
"Intent queries and agent summaries share selection authorship.",
|
|
"The devdoc slice has one binary-gold case while dev and devdoc expose the same backing search_open_platform_docs operation.",
|
|
"Catalog JSON is generated only under the ignored build temp directory and is never a repository input.",
|
|
"There are no English intent cases and only ten workflows."
|
|
]
|
|
},
|
|
"independent_test_v1": {
|
|
"state": "collecting",
|
|
"path": "scripts/testdata/tool_search_independent_test_v1.json"
|
|
}
|
|
}
|