187 lines
8.2 KiB
Python
187 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Unit tests for verify_tool_search_eval_manifest.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from collections import Counter
|
|
|
|
|
|
SCRIPT = Path(__file__).with_name("verify_tool_search_eval_manifest.py")
|
|
SPEC = importlib.util.spec_from_file_location("verify_tool_search_eval_manifest", SCRIPT)
|
|
if SPEC is None or SPEC.loader is None: # pragma: no cover
|
|
raise RuntimeError(f"cannot import {SCRIPT}")
|
|
verify = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = verify
|
|
SPEC.loader.exec_module(verify)
|
|
|
|
|
|
class ToolSearchEvalManifestTest(unittest.TestCase):
|
|
def test_language_slice(self) -> None:
|
|
self.assertEqual(verify.language_slice("给群里发文件"), "chinese_only")
|
|
self.assertEqual(
|
|
verify.language_slice("读取 baseId 对应的表格"),
|
|
"mixed_chinese_ascii",
|
|
)
|
|
self.assertEqual(verify.language_slice("send a file to the group"), "english")
|
|
|
|
def test_go_algorithm_source_freeze_matches(self) -> None:
|
|
manifest = verify.json.loads(verify.DEFAULT_MANIFEST.read_text(encoding="utf-8"))
|
|
freeze = manifest["algorithm_freeze"]
|
|
self.assertEqual(
|
|
verify.sha256_source_set(freeze["go_source_paths"]),
|
|
freeze["go_source_sha256"],
|
|
)
|
|
self.assertEqual(
|
|
verify.sha256_source_set(freeze["gate_source_paths"]),
|
|
freeze["gate_source_sha256"],
|
|
)
|
|
|
|
def test_collecting_manifest_is_valid_but_not_release_ready(self) -> None:
|
|
self.assertEqual(verify.validate_manifest(verify.DEFAULT_MANIFEST, False), [])
|
|
sealed_problems = verify.validate_manifest(verify.DEFAULT_MANIFEST, True)
|
|
self.assertTrue(
|
|
any("must be sealed" in problem for problem in sealed_problems),
|
|
sealed_problems,
|
|
)
|
|
self.assertTrue(
|
|
any("english has 0 cases" in problem for problem in sealed_problems),
|
|
sealed_problems,
|
|
)
|
|
|
|
def test_graded_equivalent_qrels_are_accepted(self) -> None:
|
|
catalog = {
|
|
"dev.search_open_platform_docs_rag",
|
|
"devdoc.search_open_platform_docs_rag",
|
|
}
|
|
case = {
|
|
"id": "devdoc-search",
|
|
"query": "查 OAuth2 接入文档",
|
|
"language": "mixed_chinese_ascii",
|
|
"qrels": [
|
|
{
|
|
"canonical": "devdoc.search_open_platform_docs_rag",
|
|
"relevance": 3,
|
|
},
|
|
{
|
|
"canonical": "dev.search_open_platform_docs_rag",
|
|
"relevance": 2,
|
|
},
|
|
],
|
|
}
|
|
problems: list[str] = []
|
|
language, workflow = verify.validate_case(case, 0, catalog, problems)
|
|
self.assertEqual(language, "mixed_chinese_ascii")
|
|
self.assertFalse(workflow)
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_forbidden_requires_alternative_gold(self) -> None:
|
|
case = {
|
|
"id": "unsafe",
|
|
"query": "删除记录",
|
|
"language": "chinese_only",
|
|
"qrels": [{"canonical": "safe.read", "relevance": 3}],
|
|
"forbidden": ["danger.delete"],
|
|
}
|
|
problems: list[str] = []
|
|
verify.validate_case(case, 0, {"safe.read", "danger.delete"}, problems)
|
|
self.assertTrue(any("alternative_gold" in problem for problem in problems))
|
|
|
|
def test_forbidden_and_alternative_gold_are_catalog_bound_and_disjoint(self) -> None:
|
|
case = {
|
|
"id": "unsafe",
|
|
"query": "删除记录",
|
|
"language": "chinese_only",
|
|
"qrels": [{"canonical": "safe.read", "relevance": 3}],
|
|
"forbidden": ["danger.delete", "unknown.tool"],
|
|
"alternative_gold": ["danger.delete"],
|
|
}
|
|
problems: list[str] = []
|
|
verify.validate_case(case, 0, {"safe.read", "danger.delete"}, problems)
|
|
self.assertTrue(any("unknown Catalog tool" in problem for problem in problems), problems)
|
|
self.assertTrue(any("must be disjoint" in problem for problem in problems), problems)
|
|
|
|
def test_independent_result_accepts_generator_envelope_payload(self) -> None:
|
|
manifest = verify.json.loads(verify.DEFAULT_MANIFEST.read_text(encoding="utf-8"))
|
|
report = {
|
|
"version": "tool-search-independent-evaluation.v1",
|
|
"catalog": {
|
|
"source_hash": manifest["proxy_v1"]["catalog_source_hash"],
|
|
"surface_hash": manifest["proxy_v1"]["catalog_surface_hash"],
|
|
},
|
|
"overall": {"cases": 3, "recall_at_5": 1.0},
|
|
"control_overall": {"cases": 3, "recall_at_5": 0.9},
|
|
"recall_at_5_delta": 0.1,
|
|
"product_cluster_recall_at_5_delta_ci_95": {"lower": 0.05, "upper": 0.15},
|
|
"language_slices": {
|
|
name: {"cases": 1, "recall_at_5": 1.0}
|
|
for name in ["chinese_only", "mixed_chinese_ascii", "english"]
|
|
},
|
|
"safety": {"forbidden_exposure_at_5": 0.0, "alternative_recall_at_5": 1.0, "sibling_exposure_at_5": 0.0},
|
|
"workflow": {"cases": 1, "complete_at_5": 1.0, "required_recall_at_5": 1.0},
|
|
}
|
|
self.assertIs(verify.unwrap_independent_result({"independent": report}), report)
|
|
problems: list[str] = []
|
|
verify.validate_independent_result(
|
|
report,
|
|
manifest["independent_test_v1"],
|
|
manifest["proxy_v1"],
|
|
Counter(chinese_only=1, mixed_chinese_ascii=1, english=1, workflow=1),
|
|
problems,
|
|
)
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_validate_manifest_unwraps_real_generator_file_shape(self) -> None:
|
|
manifest = json.loads(verify.DEFAULT_MANIFEST.read_text(encoding="utf-8"))
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
temporary = Path(directory)
|
|
qrels = temporary / "qrels.json"
|
|
qrels.write_text(json.dumps({"version": "tool-search-qrels.v1", "state": "collecting", "cases": []}), encoding="utf-8")
|
|
manifest["independent_test_v1"]["path"] = str(qrels)
|
|
manifest_path = temporary / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
result_path = temporary / "result.json"
|
|
result_path.write_text(json.dumps({"diagnostic": {}, "independent": {"version": "wrong"}}), encoding="utf-8")
|
|
problems = verify.validate_manifest(manifest_path, True, independent_result_path=result_path)
|
|
self.assertTrue(any("result version" in problem for problem in problems), problems)
|
|
|
|
def test_release_coverage_is_derived_not_self_asserted(self) -> None:
|
|
metadata = {
|
|
"chat.send": {"product_id": "chat", "effect": "write", "interface_ref": {"rpc_name": "send"}},
|
|
"chat.delete": {"product_id": "chat", "effect": "destructive", "interface_ref": {"rpc_name": "delete"}},
|
|
"doc.read": {"product_id": "doc", "effect": "read", "interface_ref": {"rpc_name": "get"}},
|
|
"doc.fetch": {"product_id": "devdoc", "effect": "read", "interface_ref": {"rpc_name": "get"}},
|
|
}
|
|
cases = [
|
|
{
|
|
"qrels": [{"canonical": "chat.send", "relevance": 3}],
|
|
"forbidden": ["chat.delete"],
|
|
"alternative_gold": ["chat.send"],
|
|
"confusion_family": ["chat.delete", "chat.send.other"],
|
|
"workflow": {"required": ["chat.send", "doc.read"]},
|
|
},
|
|
{
|
|
"qrels": [
|
|
{"canonical": "doc.read", "relevance": 3},
|
|
{"canonical": "doc.fetch", "relevance": 2},
|
|
]
|
|
},
|
|
{"qrels": [{"canonical": "chat.delete", "relevance": 3}]},
|
|
]
|
|
problems: list[str] = []
|
|
coverage = verify.derive_independent_coverage(cases, metadata, problems)
|
|
self.assertIn("all_reviewed_products", coverage)
|
|
self.assertIn("read_write_destructive", coverage)
|
|
self.assertIn("graded_equivalent_entrypoints", coverage)
|
|
self.assertNotIn("sibling_confusion", coverage)
|
|
self.assertTrue(any("confusion_family" in problem for problem in problems))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|