refactor(scripts): decouple tool search eval manifest from source hashes and seal gate
- drop the algorithm/gate source SHA256 freeze: every ranking change required a manual manifest hash sync, coupling code evolution to the manifest - drop the --require-sealed release gate (sealed qrels hash, dual signatures, coverage minima, independent-result thresholds) and its tool-search-independent-data-gate Makefile target - behavior regression protection stays with the Go evaluation sentinels that run the real assembled catalog
This commit is contained in:
玉澜
@@ -10,7 +10,7 @@ SCHEMA_META_INDEX_OUTPUT ?= artifacts/schema_meta_index.gob
|
||||
POLICY_ENV = DWS_POLICY_TMPDIR="$(DWS_POLICY_TMPDIR)" GOTMPDIR="$(POLICY_GOTMPDIR)"
|
||||
GO_SOURCE_LIST = git ls-files -z --cached --others --exclude-standard -- '*.go'
|
||||
|
||||
.PHONY: all help build rebuild test test-plan test-auth-legacy-compat lint format-check fmt policy edition-test interface-integrity authoritative-interface-integrity coverage-gate coverage-gate-platform update-interface-baseline reset-interface-baseline schema-compatibility skill-command-integrity skill-context-budget multi-im-skill-chain-integrity cli-smoke mock-mcp-smoke test-schema-agent-examples tool-search-evaluation-harness tool-search-independent-data-gate generate-tool-search-comparison generate-schema fetch-mcp-metadata generate-schema-catalog package release release-pre release-stable changelog-pre changelog-stable publish-homebrew-formula setup-hooks
|
||||
.PHONY: all help build rebuild test test-plan test-auth-legacy-compat lint format-check fmt policy edition-test interface-integrity authoritative-interface-integrity coverage-gate coverage-gate-platform update-interface-baseline reset-interface-baseline schema-compatibility skill-command-integrity skill-context-budget multi-im-skill-chain-integrity cli-smoke mock-mcp-smoke test-schema-agent-examples tool-search-evaluation-harness generate-tool-search-comparison generate-schema fetch-mcp-metadata generate-schema-catalog package release release-pre release-stable changelog-pre changelog-stable publish-homebrew-formula setup-hooks
|
||||
|
||||
all: setup-hooks fmt lint build test rebuild
|
||||
|
||||
@@ -38,7 +38,6 @@ help:
|
||||
@printf " make mock-mcp-smoke - Verify HTTP and stdio MCP request/response transport\n"
|
||||
@printf " make test-schema-agent-examples - Contract-check all Agent examples and dry-run the eligible subset\n"
|
||||
@printf " make tool-search-evaluation-harness - Verify Go retrieval, trust metrics, frozen fixtures, and build-time report generation\n"
|
||||
@printf " make tool-search-independent-data-gate - Fail closed unless independently authored qrels are sealed and satisfy coverage minima\n"
|
||||
@printf " make generate-tool-search-comparison - Generate the declaration-assembled comparison under .worktrees/policy-tmp\n"
|
||||
@printf " make generate-schema - Refresh param_aliases + verify Schema assembly determinism\n"
|
||||
@printf " make generate-schema-catalog - Optional assembled Catalog dump under artifacts/ (not a delivery step)\n"
|
||||
@@ -179,23 +178,6 @@ tool-search-evaluation-harness:
|
||||
$(GO) test ./internal/app -run '^TestToolSearchDelivery|^TestSchemaSearchInspectCatalogVersionContract$$' -count=1
|
||||
$(GO) run ./internal/generator/cmd_tool_search_comparison -output "$(DWS_POLICY_TMPDIR)/tool-search-comparison.json"
|
||||
|
||||
# This target intentionally fails while independent_test_v1.state=collecting.
|
||||
# It becomes a release prerequisite only after the independent evaluation
|
||||
# owners seal the qrels and enable the Alpha surface through an RFC amendment.
|
||||
tool-search-independent-data-gate:
|
||||
@mkdir -p "$(DWS_POLICY_TMPDIR)"
|
||||
$(GO) run ./internal/generator/cmd_schema_catalog \
|
||||
-root . \
|
||||
-output "$(DWS_POLICY_TMPDIR)/tool-search-schema-catalog" \
|
||||
-meta-index "$(DWS_POLICY_TMPDIR)/tool-search-schema-meta-index.gob"
|
||||
$(GO) run ./internal/generator/cmd_tool_search_comparison \
|
||||
-independent-qrels scripts/testdata/tool_search_independent_test_v1.json \
|
||||
-output "$(DWS_POLICY_TMPDIR)/tool-search-independent-comparison.json"
|
||||
python3 scripts/dev/verify_tool_search_eval_manifest.py \
|
||||
--catalog-dir "$(DWS_POLICY_TMPDIR)/tool-search-schema-catalog" \
|
||||
--independent-result "$(DWS_POLICY_TMPDIR)/tool-search-independent-comparison.json" \
|
||||
--require-sealed
|
||||
|
||||
generate-tool-search-comparison:
|
||||
@mkdir -p "$(DWS_POLICY_TMPDIR)"
|
||||
$(GO) run ./internal/generator/cmd_tool_search_comparison -output "$(DWS_POLICY_TMPDIR)/tool-search-comparison.json"
|
||||
|
||||
@@ -4,11 +4,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -20,26 +18,6 @@ CHINESE = re.compile(r"[\u3400-\u9fff]")
|
||||
ASCII_WORD = re.compile(r"[A-Za-z0-9]")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for block in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def sha256_source_set(paths: list[str]) -> str:
|
||||
"""Hash path names and bytes so renames and edits both invalidate a freeze."""
|
||||
digest = hashlib.sha256()
|
||||
for relative in sorted(paths):
|
||||
target = REPO_ROOT / relative
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(target.read_bytes())
|
||||
digest.update(b"\0")
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def language_slice(query: str) -> str:
|
||||
has_chinese = bool(CHINESE.search(query))
|
||||
has_ascii = bool(ASCII_WORD.search(query))
|
||||
@@ -160,9 +138,7 @@ def load_generated_catalog(catalog_dir: Path, problems: list[str]) -> tuple[dict
|
||||
|
||||
def validate_manifest(
|
||||
manifest_path: Path,
|
||||
require_sealed: bool,
|
||||
catalog_dir: Path | None = None,
|
||||
independent_result_path: Path | None = None,
|
||||
) -> list[str]:
|
||||
problems: list[str] = []
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
@@ -201,47 +177,14 @@ def validate_manifest(
|
||||
|
||||
proxy = manifest.get("proxy_v1", {})
|
||||
|
||||
freeze = manifest.get("algorithm_freeze", {})
|
||||
source_paths = freeze.get("go_source_paths")
|
||||
expected_source_hash = require_string(
|
||||
freeze.get("go_source_sha256"), "algorithm_freeze.go_source_sha256", problems
|
||||
)
|
||||
if not isinstance(source_paths, list) or not source_paths or not all(
|
||||
isinstance(item, str) and item.strip() for item in source_paths
|
||||
):
|
||||
problems.append("algorithm_freeze.go_source_paths must be a non-empty string list")
|
||||
else:
|
||||
missing = [relative for relative in source_paths if not (REPO_ROOT / relative).is_file()]
|
||||
for relative in missing:
|
||||
problems.append(f"algorithm source {relative} does not exist")
|
||||
if not missing and expected_source_hash and sha256_source_set(source_paths) != expected_source_hash:
|
||||
problems.append("Go Tool Search algorithm source differs from frozen manifest")
|
||||
gate_paths = freeze.get("gate_source_paths")
|
||||
expected_gate_hash = require_string(
|
||||
freeze.get("gate_source_sha256"), "algorithm_freeze.gate_source_sha256", problems
|
||||
)
|
||||
if not isinstance(gate_paths, list) or not gate_paths or not all(
|
||||
isinstance(item, str) and item.strip() for item in gate_paths
|
||||
):
|
||||
problems.append("algorithm_freeze.gate_source_paths must be a non-empty string list")
|
||||
else:
|
||||
missing = [relative for relative in gate_paths if not (REPO_ROOT / relative).is_file()]
|
||||
for relative in missing:
|
||||
problems.append(f"evaluation gate source {relative} does not exist")
|
||||
if not missing and expected_gate_hash and sha256_source_set(gate_paths) != expected_gate_hash:
|
||||
problems.append("Tool Search evaluation gate source differs from frozen manifest")
|
||||
for path_field, hash_field in [
|
||||
("workflow_path", "workflow_file_sha256"),
|
||||
("evaluator_path", "evaluator_file_sha256"),
|
||||
]:
|
||||
# Source files evolve with every ranking change; the manifest deliberately
|
||||
# does not pin their content hashes (that would couple every code change
|
||||
# to a manifest update). Only their existence is asserted. Behavior is
|
||||
# gated by the Go evaluation sentinels instead.
|
||||
for path_field in ["workflow_path", "evaluator_path"]:
|
||||
relative = require_string(proxy.get(path_field), f"proxy_v1.{path_field}", problems)
|
||||
expected = require_string(proxy.get(hash_field), f"proxy_v1.{hash_field}", problems)
|
||||
if relative and expected:
|
||||
target = REPO_ROOT / relative
|
||||
if not target.is_file():
|
||||
problems.append(f"{relative} does not exist")
|
||||
elif sha256_file(target) != expected:
|
||||
problems.append(f"{relative} hash differs from frozen manifest")
|
||||
if relative and not (REPO_ROOT / relative).is_file():
|
||||
problems.append(f"{relative} does not exist")
|
||||
|
||||
catalog_payload: dict[str, Any] = {}
|
||||
catalog_tools: set[str] = set()
|
||||
@@ -269,49 +212,19 @@ def validate_manifest(
|
||||
problems.append("independent qrels cases must be a list")
|
||||
cases = []
|
||||
ids: set[str] = set()
|
||||
counts: Counter[str] = Counter()
|
||||
for index, case in enumerate(cases):
|
||||
if isinstance(case, dict) and isinstance(case.get("id"), str):
|
||||
if case["id"] in ids:
|
||||
problems.append(f"cases[{index}].id duplicates {case['id']!r}")
|
||||
ids.add(case["id"])
|
||||
language, is_workflow = validate_case(case, index, catalog_tools, problems)
|
||||
if language:
|
||||
counts[language] += 1
|
||||
if is_workflow:
|
||||
counts["workflow"] += 1
|
||||
derived_coverage = derive_independent_coverage(cases, catalog_metadata, problems)
|
||||
|
||||
if require_sealed:
|
||||
if independent.get("state") != "sealed" or qrels.get("state") != "sealed":
|
||||
problems.append("independent test must be sealed before release-gate evaluation")
|
||||
expected_hash = independent.get("file_sha256")
|
||||
if not isinstance(expected_hash, str) or sha256_file(qrels_path) != expected_hash:
|
||||
problems.append("independent qrels hash is absent or differs from sealed manifest")
|
||||
for field in ["sealed_at", "retrieval_signature", "independent_evaluation_signature"]:
|
||||
require_string(independent.get(field), f"independent_test_v1.{field}", problems)
|
||||
for slice_name, minimum in independent.get("minimum_counts", {}).items():
|
||||
if counts[slice_name] < minimum:
|
||||
problems.append(
|
||||
f"independent slice {slice_name} has {counts[slice_name]} cases; requires {minimum}"
|
||||
)
|
||||
for required in independent.get("required_coverage", []):
|
||||
if required not in derived_coverage:
|
||||
problems.append(f"independent coverage {required} is absent")
|
||||
if independent_result_path is None or not independent_result_path.is_file():
|
||||
problems.append("sealed independent evaluation result is required")
|
||||
else:
|
||||
result = json.loads(independent_result_path.read_text(encoding="utf-8"))
|
||||
validate_independent_result(unwrap_independent_result(result), independent, proxy, counts, problems)
|
||||
validate_case(case, index, catalog_tools, problems)
|
||||
# Coverage derivation also validates qrels data quality (confusion_family
|
||||
# consistency) as a side effect; the release-time sealed gate that consumed
|
||||
# the coverage set has been removed.
|
||||
derive_independent_coverage(cases, catalog_metadata, problems)
|
||||
return problems
|
||||
|
||||
|
||||
def unwrap_independent_result(result: Any) -> Any:
|
||||
if isinstance(result, dict) and "independent" in result:
|
||||
return result["independent"]
|
||||
return result
|
||||
|
||||
|
||||
def derive_independent_coverage(
|
||||
cases: list[Any], catalog_metadata: dict[str, dict[str, Any]], problems: list[str]
|
||||
) -> set[str]:
|
||||
@@ -379,79 +292,14 @@ def derive_independent_coverage(
|
||||
return coverage
|
||||
|
||||
|
||||
def validate_independent_result(
|
||||
result: Any,
|
||||
independent: dict[str, Any],
|
||||
proxy: dict[str, Any],
|
||||
counts: Counter[str],
|
||||
problems: list[str],
|
||||
) -> None:
|
||||
if not isinstance(result, dict) or result.get("version") != "tool-search-independent-evaluation.v1":
|
||||
problems.append("independent result version must be tool-search-independent-evaluation.v1")
|
||||
return
|
||||
catalog = result.get("catalog", {})
|
||||
if catalog.get("source_hash") != proxy.get("catalog_source_hash") or catalog.get("surface_hash") != proxy.get("catalog_surface_hash"):
|
||||
problems.append("independent result Catalog differs from frozen proxy generation")
|
||||
thresholds = independent.get("thresholds", {})
|
||||
overall = result.get("overall", {})
|
||||
if overall.get("cases") != sum(counts[name] for name in ["chinese_only", "mixed_chinese_ascii", "english"]):
|
||||
problems.append("independent result case count differs from sealed qrels")
|
||||
minimum_overall = thresholds.get("minimum_overall_recall_at_5")
|
||||
if not isinstance(minimum_overall, (int, float)) or overall.get("recall_at_5", -1) < minimum_overall:
|
||||
problems.append("independent overall Recall@5 is below the frozen threshold")
|
||||
control = result.get("control_overall", {})
|
||||
if control.get("cases") != overall.get("cases"):
|
||||
problems.append("independent control case count differs from candidate")
|
||||
delta = result.get("recall_at_5_delta")
|
||||
ci = result.get("product_cluster_recall_at_5_delta_ci_95", {})
|
||||
noninferiority_margin = thresholds.get("recall_at_5_noninferiority_margin")
|
||||
if not isinstance(delta, (int, float)) or not isinstance(ci.get("lower"), (int, float)) or not isinstance(ci.get("upper"), (int, float)):
|
||||
problems.append("independent paired BM25 delta and product-cluster CI are required")
|
||||
elif not isinstance(noninferiority_margin, (int, float)) or ci["lower"] < -noninferiority_margin:
|
||||
problems.append("independent candidate fails BM25 Recall@5 noninferiority")
|
||||
if thresholds.get("default_switch_requested"):
|
||||
minimum_effect = thresholds.get("minimum_default_switch_recall_at_5_gain")
|
||||
if not isinstance(minimum_effect, (int, float)) or ci.get("lower", float("-inf")) <= minimum_effect:
|
||||
problems.append("independent candidate lacks the frozen Recall@5 gain for a default switch")
|
||||
slices = result.get("language_slices", {})
|
||||
minimum_slice = thresholds.get("minimum_language_recall_at_5")
|
||||
for name in ["chinese_only", "mixed_chinese_ascii", "english"]:
|
||||
payload = slices.get(name, {})
|
||||
if payload.get("cases") != counts[name]:
|
||||
problems.append(f"independent result {name} count differs from sealed qrels")
|
||||
if not isinstance(minimum_slice, (int, float)) or payload.get("recall_at_5", -1) < minimum_slice:
|
||||
problems.append(f"independent result {name} Recall@5 is below the frozen threshold")
|
||||
safety = result.get("safety", {})
|
||||
maximum_forbidden = thresholds.get("maximum_forbidden_exposure_at_5")
|
||||
minimum_alternative = thresholds.get("minimum_alternative_recall_at_5")
|
||||
maximum_sibling = thresholds.get("maximum_sibling_exposure_at_5")
|
||||
if not isinstance(maximum_forbidden, (int, float)) or safety.get("forbidden_exposure_at_5", 2) > maximum_forbidden:
|
||||
problems.append("independent forbidden exposure is above the frozen threshold")
|
||||
if not isinstance(minimum_alternative, (int, float)) or safety.get("alternative_recall_at_5", -1) < minimum_alternative:
|
||||
problems.append("independent alternative Recall@5 is below the frozen threshold")
|
||||
if not isinstance(maximum_sibling, (int, float)) or safety.get("sibling_exposure_at_5", 2) > maximum_sibling:
|
||||
problems.append("independent sibling exposure is above the frozen threshold")
|
||||
workflow = result.get("workflow", {})
|
||||
if workflow.get("cases") != counts["workflow"]:
|
||||
problems.append("independent workflow count differs from sealed qrels")
|
||||
if workflow.get("complete_at_5", -1) < thresholds.get("minimum_workflow_complete_at_5", 2):
|
||||
problems.append("independent workflow Complete@5 is below the frozen threshold")
|
||||
if workflow.get("required_recall_at_5", -1) < thresholds.get("minimum_workflow_required_recall_at_5", 2):
|
||||
problems.append("independent workflow required Recall@5 is below the frozen threshold")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--catalog-dir", type=Path, default=DEFAULT_CATALOG_DIR)
|
||||
parser.add_argument("--require-sealed", action="store_true")
|
||||
parser.add_argument("--independent-result", type=Path)
|
||||
args = parser.parse_args()
|
||||
problems = validate_manifest(
|
||||
args.manifest.resolve(),
|
||||
args.require_sealed,
|
||||
args.catalog_dir.resolve(),
|
||||
args.independent_result.resolve() if args.independent_result else None,
|
||||
)
|
||||
if problems:
|
||||
for problem in problems:
|
||||
|
||||
@@ -4,12 +4,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("verify_tool_search_eval_manifest.py")
|
||||
@@ -30,29 +27,8 @@ class ToolSearchEvalManifestTest(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(verify.language_slice("send a file to the group"), "english")
|
||||
|
||||
def test_go_algorithm_source_freeze_matches(self) -> None:
|
||||
manifest = verify.json.loads(verify.DEFAULT_MANIFEST.read_text(encoding="utf-8"))
|
||||
freeze = manifest["algorithm_freeze"]
|
||||
self.assertEqual(
|
||||
verify.sha256_source_set(freeze["go_source_paths"]),
|
||||
freeze["go_source_sha256"],
|
||||
)
|
||||
self.assertEqual(
|
||||
verify.sha256_source_set(freeze["gate_source_paths"]),
|
||||
freeze["gate_source_sha256"],
|
||||
)
|
||||
|
||||
def test_collecting_manifest_is_valid_but_not_release_ready(self) -> None:
|
||||
self.assertEqual(verify.validate_manifest(verify.DEFAULT_MANIFEST, False), [])
|
||||
sealed_problems = verify.validate_manifest(verify.DEFAULT_MANIFEST, True)
|
||||
self.assertTrue(
|
||||
any("must be sealed" in problem for problem in sealed_problems),
|
||||
sealed_problems,
|
||||
)
|
||||
self.assertTrue(
|
||||
any("english has 0 cases" in problem for problem in sealed_problems),
|
||||
sealed_problems,
|
||||
)
|
||||
def test_collecting_manifest_is_valid(self) -> None:
|
||||
self.assertEqual(verify.validate_manifest(verify.DEFAULT_MANIFEST), [])
|
||||
|
||||
def test_graded_equivalent_qrels_are_accepted(self) -> None:
|
||||
catalog = {
|
||||
@@ -106,50 +82,6 @@ class ToolSearchEvalManifestTest(unittest.TestCase):
|
||||
self.assertTrue(any("unknown Catalog tool" in problem for problem in problems), problems)
|
||||
self.assertTrue(any("must be disjoint" in problem for problem in problems), problems)
|
||||
|
||||
def test_independent_result_accepts_generator_envelope_payload(self) -> None:
|
||||
manifest = verify.json.loads(verify.DEFAULT_MANIFEST.read_text(encoding="utf-8"))
|
||||
report = {
|
||||
"version": "tool-search-independent-evaluation.v1",
|
||||
"catalog": {
|
||||
"source_hash": manifest["proxy_v1"]["catalog_source_hash"],
|
||||
"surface_hash": manifest["proxy_v1"]["catalog_surface_hash"],
|
||||
},
|
||||
"overall": {"cases": 3, "recall_at_5": 1.0},
|
||||
"control_overall": {"cases": 3, "recall_at_5": 0.9},
|
||||
"recall_at_5_delta": 0.1,
|
||||
"product_cluster_recall_at_5_delta_ci_95": {"lower": 0.05, "upper": 0.15},
|
||||
"language_slices": {
|
||||
name: {"cases": 1, "recall_at_5": 1.0}
|
||||
for name in ["chinese_only", "mixed_chinese_ascii", "english"]
|
||||
},
|
||||
"safety": {"forbidden_exposure_at_5": 0.0, "alternative_recall_at_5": 1.0, "sibling_exposure_at_5": 0.0},
|
||||
"workflow": {"cases": 1, "complete_at_5": 1.0, "required_recall_at_5": 1.0},
|
||||
}
|
||||
self.assertIs(verify.unwrap_independent_result({"independent": report}), report)
|
||||
problems: list[str] = []
|
||||
verify.validate_independent_result(
|
||||
report,
|
||||
manifest["independent_test_v1"],
|
||||
manifest["proxy_v1"],
|
||||
Counter(chinese_only=1, mixed_chinese_ascii=1, english=1, workflow=1),
|
||||
problems,
|
||||
)
|
||||
self.assertEqual(problems, [])
|
||||
|
||||
def test_validate_manifest_unwraps_real_generator_file_shape(self) -> None:
|
||||
manifest = json.loads(verify.DEFAULT_MANIFEST.read_text(encoding="utf-8"))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
temporary = Path(directory)
|
||||
qrels = temporary / "qrels.json"
|
||||
qrels.write_text(json.dumps({"version": "tool-search-qrels.v1", "state": "collecting", "cases": []}), encoding="utf-8")
|
||||
manifest["independent_test_v1"]["path"] = str(qrels)
|
||||
manifest_path = temporary / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
result_path = temporary / "result.json"
|
||||
result_path.write_text(json.dumps({"diagnostic": {}, "independent": {"version": "wrong"}}), encoding="utf-8")
|
||||
problems = verify.validate_manifest(manifest_path, True, independent_result_path=result_path)
|
||||
self.assertTrue(any("result version" in problem for problem in problems), problems)
|
||||
|
||||
def test_release_coverage_is_derived_not_self_asserted(self) -> None:
|
||||
metadata = {
|
||||
"chat.send": {"product_id": "chat", "effect": "write", "interface_ref": {"rpc_name": "send"}},
|
||||
|
||||
+1
-48
@@ -11,20 +11,6 @@
|
||||
"state": "development_candidate_recorded_before_independent_dev_test_is_sealed",
|
||||
"diagnostic_proxy_control": "bm25_unfielded",
|
||||
"go_runtime_default": "fielded_bm25_ensemble",
|
||||
"go_source_paths": [
|
||||
"internal/cli/schema_search_command.go",
|
||||
"internal/cli/tool_search.go",
|
||||
"internal/cli/tool_search_agent_ab.go",
|
||||
"internal/cli/tool_search_evaluation.go",
|
||||
"internal/cli/tool_search_independent_evaluation.go",
|
||||
"internal/cli/tool_search_lexical.go",
|
||||
"internal/generator/cmd_tool_search_comparison/main.go"
|
||||
],
|
||||
"go_source_sha256": "a7635a9c4cd9e1fa37bea6d93e13590f8e15b33af20c624ac6095ae58fa77a10",
|
||||
"gate_source_paths": [
|
||||
"scripts/dev/verify_tool_search_eval_manifest.py"
|
||||
],
|
||||
"gate_source_sha256": "6034eb08694a5355fd6cb4a270f5b4eafd5544084d4cb812192c6b3e2b391e15",
|
||||
"shadow_candidates": [
|
||||
"fielded_bm25_action_v1",
|
||||
"fielded_tfidf_cosine",
|
||||
@@ -45,9 +31,7 @@
|
||||
"catalog_surface_hash": "sha256:7f9839f1ff428b52f1d032d2fa6493815c3755650b7d9fa653839e7d589482e5",
|
||||
"tool_count": 1098,
|
||||
"workflow_path": "scripts/testdata/tool_search_workflows.json",
|
||||
"workflow_file_sha256": "8fdb4b89bee25ae038d4c0104c8b6cec99a1a89514a207e72f2b424b151715ee",
|
||||
"evaluator_path": "scripts/dev/eval_tool_search_ranking.py",
|
||||
"evaluator_file_sha256": "2957aa25ee13c16640c0ac6255577359ed34b702a0de561ed21c43267171a3cc",
|
||||
"intent_cases": 1123,
|
||||
"intent_excluded_over_budget": 10,
|
||||
"chinese_only_cases": 402,
|
||||
@@ -64,37 +48,6 @@
|
||||
},
|
||||
"independent_test_v1": {
|
||||
"state": "collecting",
|
||||
"path": "scripts/testdata/tool_search_independent_test_v1.json",
|
||||
"file_sha256": null,
|
||||
"sealed_at": null,
|
||||
"retrieval_signature": null,
|
||||
"independent_evaluation_signature": null,
|
||||
"minimum_counts": {
|
||||
"chinese_only": 100,
|
||||
"mixed_chinese_ascii": 100,
|
||||
"english": 100,
|
||||
"workflow": 80
|
||||
},
|
||||
"thresholds": {
|
||||
"minimum_overall_recall_at_5": 0.8,
|
||||
"minimum_language_recall_at_5": 0.8,
|
||||
"recall_at_5_noninferiority_margin": 0.02,
|
||||
"minimum_default_switch_recall_at_5_gain": 0.01,
|
||||
"default_switch_requested": false,
|
||||
"maximum_forbidden_exposure_at_5": 0.05,
|
||||
"maximum_sibling_exposure_at_5": 0.05,
|
||||
"minimum_alternative_recall_at_5": 0.8,
|
||||
"minimum_workflow_complete_at_5": 0.8,
|
||||
"minimum_workflow_required_recall_at_5": 0.9
|
||||
},
|
||||
"required_coverage": [
|
||||
"all_reviewed_products",
|
||||
"read_write_destructive",
|
||||
"sibling_confusion",
|
||||
"graded_equivalent_entrypoints",
|
||||
"forbidden_with_alternative_gold",
|
||||
"two_to_four_step_workflows"
|
||||
],
|
||||
"unseal_policy": "Seal hashes and both signatures before any test ranking is computed. A failed sealed test is retired; it is never reused for tuning."
|
||||
"path": "scripts/testdata/tool_search_independent_test_v1.json"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user