466 lines
23 KiB
Python
466 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify frozen Tool Search evaluation inputs and independent qrels gates."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import subprocess
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
DEFAULT_MANIFEST = REPO_ROOT / "scripts/testdata/tool_search_eval_manifest.json"
|
|
DEFAULT_CATALOG_DIR = REPO_ROOT / ".worktrees/policy-tmp/tool-search-schema-catalog"
|
|
CHINESE = re.compile(r"[\u3400-\u9fff]")
|
|
ASCII_WORD = re.compile(r"[A-Za-z0-9]")
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def sha256_source_set(paths: list[str]) -> str:
|
|
"""Hash path names and bytes so renames and edits both invalidate a freeze."""
|
|
digest = hashlib.sha256()
|
|
for relative in sorted(paths):
|
|
target = REPO_ROOT / relative
|
|
digest.update(relative.encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(target.read_bytes())
|
|
digest.update(b"\0")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def language_slice(query: str) -> str:
|
|
has_chinese = bool(CHINESE.search(query))
|
|
has_ascii = bool(ASCII_WORD.search(query))
|
|
if has_chinese and has_ascii:
|
|
return "mixed_chinese_ascii"
|
|
if has_chinese:
|
|
return "chinese_only"
|
|
return "english"
|
|
|
|
|
|
def require_string(value: Any, field: str, problems: list[str]) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
problems.append(f"{field} must be a non-empty string")
|
|
return ""
|
|
return value.strip()
|
|
|
|
|
|
def validate_case(case: Any, index: int, catalog_tools: set[str], problems: list[str]) -> tuple[str, bool]:
|
|
prefix = f"cases[{index}]"
|
|
if not isinstance(case, dict):
|
|
problems.append(f"{prefix} must be an object")
|
|
return "", False
|
|
require_string(case.get("id"), f"{prefix}.id", problems)
|
|
query = require_string(case.get("query"), f"{prefix}.query", problems)
|
|
declared_language = require_string(case.get("language"), f"{prefix}.language", problems)
|
|
actual_language = language_slice(query) if query else ""
|
|
if declared_language and declared_language != actual_language:
|
|
problems.append(
|
|
f"{prefix}.language={declared_language!r} does not match {actual_language!r}"
|
|
)
|
|
qrels = case.get("qrels")
|
|
if not isinstance(qrels, list) or not qrels:
|
|
problems.append(f"{prefix}.qrels must be a non-empty list")
|
|
else:
|
|
seen: set[str] = set()
|
|
for qrel_index, qrel in enumerate(qrels):
|
|
qrel_prefix = f"{prefix}.qrels[{qrel_index}]"
|
|
if not isinstance(qrel, dict):
|
|
problems.append(f"{qrel_prefix} must be an object")
|
|
continue
|
|
canonical = require_string(qrel.get("canonical"), f"{qrel_prefix}.canonical", problems)
|
|
relevance = qrel.get("relevance")
|
|
if canonical in seen:
|
|
problems.append(f"{qrel_prefix}.canonical duplicates {canonical!r}")
|
|
seen.add(canonical)
|
|
if canonical and canonical not in catalog_tools:
|
|
problems.append(f"{qrel_prefix}.canonical {canonical!r} is not in the Catalog")
|
|
if not isinstance(relevance, int) or relevance not in {1, 2, 3}:
|
|
problems.append(f"{qrel_prefix}.relevance must be 1, 2 or 3")
|
|
forbidden = case.get("forbidden", [])
|
|
alternatives = case.get("alternative_gold", [])
|
|
if forbidden:
|
|
valid_forbidden = isinstance(forbidden, list) and all(isinstance(item, str) and item.strip() for item in forbidden)
|
|
if not valid_forbidden:
|
|
problems.append(f"{prefix}.forbidden must be a non-empty string list")
|
|
elif len(set(forbidden)) != len(forbidden):
|
|
problems.append(f"{prefix}.forbidden must not contain duplicates")
|
|
elif any(item not in catalog_tools for item in forbidden):
|
|
problems.append(f"{prefix}.forbidden contains an unknown Catalog tool")
|
|
valid_alternatives = isinstance(alternatives, list) and bool(alternatives) and all(
|
|
isinstance(item, str) and item.strip() for item in alternatives
|
|
)
|
|
if not valid_alternatives:
|
|
problems.append(f"{prefix}.alternative_gold is required when forbidden is non-empty")
|
|
elif len(set(alternatives)) != len(alternatives):
|
|
problems.append(f"{prefix}.alternative_gold must not contain duplicates")
|
|
elif any(item not in catalog_tools for item in alternatives):
|
|
problems.append(f"{prefix}.alternative_gold contains an unknown Catalog tool")
|
|
if valid_forbidden and valid_alternatives and set(forbidden).intersection(alternatives):
|
|
problems.append(f"{prefix}.forbidden and alternative_gold must be disjoint")
|
|
elif alternatives:
|
|
problems.append(f"{prefix}.alternative_gold requires a non-empty forbidden list")
|
|
workflow = case.get("workflow")
|
|
is_workflow = workflow is not None
|
|
if is_workflow:
|
|
if not isinstance(workflow, dict):
|
|
problems.append(f"{prefix}.workflow must be an object")
|
|
else:
|
|
required = workflow.get("required", [])
|
|
if not isinstance(required, list) or not 2 <= len(required) <= 4:
|
|
problems.append(f"{prefix}.workflow.required must contain 2 to 4 tools")
|
|
elif any(item not in catalog_tools for item in required):
|
|
problems.append(f"{prefix}.workflow.required contains an unknown Catalog tool")
|
|
subqueries = workflow.get("subqueries", [])
|
|
if subqueries and (
|
|
not isinstance(subqueries, list)
|
|
or not all(isinstance(item, str) and item.strip() for item in subqueries)
|
|
):
|
|
problems.append(f"{prefix}.workflow.subqueries must be a non-empty string list when present")
|
|
return actual_language, is_workflow
|
|
|
|
|
|
def load_generated_catalog(catalog_dir: Path, problems: list[str]) -> tuple[dict[str, Any], set[str], dict[str, dict[str, Any]]]:
|
|
envelope_path = catalog_dir / "catalog.json"
|
|
tools_dir = catalog_dir / "tools"
|
|
if not envelope_path.is_file():
|
|
problems.append(f"generated Catalog envelope does not exist: {envelope_path}")
|
|
return {}, set(), {}
|
|
if not tools_dir.is_dir():
|
|
problems.append(f"generated Catalog tools directory does not exist: {tools_dir}")
|
|
return {}, set(), {}
|
|
envelope = json.loads(envelope_path.read_text(encoding="utf-8"))
|
|
tools: set[str] = set()
|
|
metadata: dict[str, dict[str, Any]] = {}
|
|
for shard_path in sorted(tools_dir.glob("*.json")):
|
|
shard = json.loads(shard_path.read_text(encoding="utf-8"))
|
|
shard_tools = shard.get("tools")
|
|
if not isinstance(shard_tools, dict):
|
|
problems.append(f"generated Catalog shard has no tools object: {shard_path}")
|
|
continue
|
|
overlap = tools.intersection(shard_tools)
|
|
if overlap:
|
|
problems.append(f"generated Catalog shards duplicate tools: {sorted(overlap)}")
|
|
tools.update(shard_tools)
|
|
metadata.update(shard_tools)
|
|
return envelope, tools, metadata
|
|
|
|
|
|
def validate_manifest(
|
|
manifest_path: Path,
|
|
require_sealed: bool,
|
|
catalog_dir: Path | None = None,
|
|
independent_result_path: Path | None = None,
|
|
) -> list[str]:
|
|
problems: list[str] = []
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
if manifest.get("version") != "tool-search-eval-manifest.v1":
|
|
problems.append("manifest version must be tool-search-eval-manifest.v1")
|
|
base_commit = require_string(manifest.get("base_repository_commit"), "base_repository_commit", problems)
|
|
if base_commit:
|
|
exists = subprocess.run(
|
|
["git", "cat-file", "-e", f"{base_commit}^{{commit}}"],
|
|
cwd=REPO_ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
check=False,
|
|
)
|
|
if exists.returncode != 0:
|
|
problems.append("base_repository_commit does not exist")
|
|
else:
|
|
ancestor = subprocess.run(
|
|
["git", "merge-base", "--is-ancestor", base_commit, "HEAD"],
|
|
cwd=REPO_ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
check=False,
|
|
)
|
|
if ancestor.returncode != 0:
|
|
problems.append("base_repository_commit is not an ancestor of HEAD")
|
|
|
|
owners = manifest.get("owners", {})
|
|
retrieval_owner = require_string(owners.get("retrieval"), "owners.retrieval", problems)
|
|
evaluation_owner = require_string(
|
|
owners.get("independent_evaluation"), "owners.independent_evaluation", problems
|
|
)
|
|
if retrieval_owner and retrieval_owner == evaluation_owner:
|
|
problems.append("retrieval and independent evaluation owners must differ")
|
|
require_string(owners.get("separation_rule"), "owners.separation_rule", problems)
|
|
|
|
proxy = manifest.get("proxy_v1", {})
|
|
|
|
freeze = manifest.get("algorithm_freeze", {})
|
|
source_paths = freeze.get("go_source_paths")
|
|
expected_source_hash = require_string(
|
|
freeze.get("go_source_sha256"), "algorithm_freeze.go_source_sha256", problems
|
|
)
|
|
if not isinstance(source_paths, list) or not source_paths or not all(
|
|
isinstance(item, str) and item.strip() for item in source_paths
|
|
):
|
|
problems.append("algorithm_freeze.go_source_paths must be a non-empty string list")
|
|
else:
|
|
missing = [relative for relative in source_paths if not (REPO_ROOT / relative).is_file()]
|
|
for relative in missing:
|
|
problems.append(f"algorithm source {relative} does not exist")
|
|
if not missing and expected_source_hash and sha256_source_set(source_paths) != expected_source_hash:
|
|
problems.append("Go Tool Search algorithm source differs from frozen manifest")
|
|
gate_paths = freeze.get("gate_source_paths")
|
|
expected_gate_hash = require_string(
|
|
freeze.get("gate_source_sha256"), "algorithm_freeze.gate_source_sha256", problems
|
|
)
|
|
if not isinstance(gate_paths, list) or not gate_paths or not all(
|
|
isinstance(item, str) and item.strip() for item in gate_paths
|
|
):
|
|
problems.append("algorithm_freeze.gate_source_paths must be a non-empty string list")
|
|
else:
|
|
missing = [relative for relative in gate_paths if not (REPO_ROOT / relative).is_file()]
|
|
for relative in missing:
|
|
problems.append(f"evaluation gate source {relative} does not exist")
|
|
if not missing and expected_gate_hash and sha256_source_set(gate_paths) != expected_gate_hash:
|
|
problems.append("Tool Search evaluation gate source differs from frozen manifest")
|
|
for path_field, hash_field in [
|
|
("workflow_path", "workflow_file_sha256"),
|
|
("evaluator_path", "evaluator_file_sha256"),
|
|
]:
|
|
relative = require_string(proxy.get(path_field), f"proxy_v1.{path_field}", problems)
|
|
expected = require_string(proxy.get(hash_field), f"proxy_v1.{hash_field}", problems)
|
|
if relative and expected:
|
|
target = REPO_ROOT / relative
|
|
if not target.is_file():
|
|
problems.append(f"{relative} does not exist")
|
|
elif sha256_file(target) != expected:
|
|
problems.append(f"{relative} hash differs from frozen manifest")
|
|
|
|
catalog_payload: dict[str, Any] = {}
|
|
catalog_tools: set[str] = set()
|
|
catalog_metadata: dict[str, dict[str, Any]] = {}
|
|
if catalog_dir is not None:
|
|
catalog_payload, catalog_tools, catalog_metadata = load_generated_catalog(catalog_dir, problems)
|
|
if len(catalog_tools) != proxy.get("tool_count"):
|
|
problems.append("proxy_v1.tool_count differs from generated Catalog")
|
|
if catalog_payload.get("source_hash") != proxy.get("catalog_source_hash"):
|
|
problems.append("proxy_v1.catalog_source_hash differs from generated Catalog")
|
|
if catalog_payload.get("surface_hash") != proxy.get("catalog_surface_hash"):
|
|
problems.append("proxy_v1.catalog_surface_hash differs from generated Catalog")
|
|
|
|
independent = manifest.get("independent_test_v1", {})
|
|
qrels_relative = require_string(independent.get("path"), "independent_test_v1.path", problems)
|
|
qrels_path = REPO_ROOT / qrels_relative
|
|
if not qrels_path.is_file():
|
|
problems.append(f"{qrels_relative} does not exist")
|
|
return problems
|
|
qrels = json.loads(qrels_path.read_text(encoding="utf-8"))
|
|
if qrels.get("version") != "tool-search-qrels.v1":
|
|
problems.append("independent qrels version must be tool-search-qrels.v1")
|
|
cases = qrels.get("cases")
|
|
if not isinstance(cases, list):
|
|
problems.append("independent qrels cases must be a list")
|
|
cases = []
|
|
ids: set[str] = set()
|
|
counts: Counter[str] = Counter()
|
|
for index, case in enumerate(cases):
|
|
if isinstance(case, dict) and isinstance(case.get("id"), str):
|
|
if case["id"] in ids:
|
|
problems.append(f"cases[{index}].id duplicates {case['id']!r}")
|
|
ids.add(case["id"])
|
|
language, is_workflow = validate_case(case, index, catalog_tools, problems)
|
|
if language:
|
|
counts[language] += 1
|
|
if is_workflow:
|
|
counts["workflow"] += 1
|
|
derived_coverage = derive_independent_coverage(cases, catalog_metadata, problems)
|
|
|
|
if require_sealed:
|
|
if independent.get("state") != "sealed" or qrels.get("state") != "sealed":
|
|
problems.append("independent test must be sealed before release-gate evaluation")
|
|
expected_hash = independent.get("file_sha256")
|
|
if not isinstance(expected_hash, str) or sha256_file(qrels_path) != expected_hash:
|
|
problems.append("independent qrels hash is absent or differs from sealed manifest")
|
|
for field in ["sealed_at", "retrieval_signature", "independent_evaluation_signature"]:
|
|
require_string(independent.get(field), f"independent_test_v1.{field}", problems)
|
|
for slice_name, minimum in independent.get("minimum_counts", {}).items():
|
|
if counts[slice_name] < minimum:
|
|
problems.append(
|
|
f"independent slice {slice_name} has {counts[slice_name]} cases; requires {minimum}"
|
|
)
|
|
for required in independent.get("required_coverage", []):
|
|
if required not in derived_coverage:
|
|
problems.append(f"independent coverage {required} is absent")
|
|
if independent_result_path is None or not independent_result_path.is_file():
|
|
problems.append("sealed independent evaluation result is required")
|
|
else:
|
|
result = json.loads(independent_result_path.read_text(encoding="utf-8"))
|
|
validate_independent_result(unwrap_independent_result(result), independent, proxy, counts, problems)
|
|
return problems
|
|
|
|
|
|
def unwrap_independent_result(result: Any) -> Any:
|
|
if isinstance(result, dict) and "independent" in result:
|
|
return result["independent"]
|
|
return result
|
|
|
|
|
|
def derive_independent_coverage(
|
|
cases: list[Any], catalog_metadata: dict[str, dict[str, Any]], problems: list[str]
|
|
) -> set[str]:
|
|
"""Derive release coverage from qrels and Catalog facts, never self-asserted labels."""
|
|
coverage: set[str] = set()
|
|
if not catalog_metadata:
|
|
return coverage
|
|
catalog_products = {str(tool.get("product_id", "")) for tool in catalog_metadata.values()}
|
|
judged: set[str] = set()
|
|
effects: set[str] = set()
|
|
workflow_ok = False
|
|
graded_equivalent = False
|
|
forbidden_alternative = False
|
|
sibling_confusion = False
|
|
for index, case in enumerate(cases):
|
|
if not isinstance(case, dict):
|
|
continue
|
|
canonicals = [
|
|
qrel.get("canonical") for qrel in case.get("qrels", []) if isinstance(qrel, dict)
|
|
]
|
|
for canonical in canonicals:
|
|
if canonical in catalog_metadata:
|
|
judged.add(canonical)
|
|
effects.add(str(catalog_metadata[canonical].get("effect", "")))
|
|
if len(canonicals) >= 2:
|
|
backing = {
|
|
json.dumps(catalog_metadata.get(item, {}).get("interface_ref", {}), sort_keys=True)
|
|
for item in canonicals
|
|
}
|
|
relevances = {
|
|
qrel.get("relevance") for qrel in case.get("qrels", []) if isinstance(qrel, dict)
|
|
}
|
|
if len(backing) == 1 and "{}" not in backing and len(relevances) >= 2:
|
|
graded_equivalent = True
|
|
forbidden = case.get("forbidden", [])
|
|
alternatives = case.get("alternative_gold", [])
|
|
if forbidden and alternatives:
|
|
forbidden_alternative = True
|
|
workflow = case.get("workflow")
|
|
if isinstance(workflow, dict) and 2 <= len(workflow.get("required", [])) <= 4:
|
|
workflow_ok = True
|
|
confusion = case.get("confusion_family", [])
|
|
if confusion:
|
|
if not isinstance(confusion, list) or len(confusion) < 2 or any(item not in catalog_metadata for item in confusion):
|
|
problems.append(f"cases[{index}].confusion_family must contain at least two Catalog tools")
|
|
elif set(confusion).intersection(canonicals):
|
|
problems.append(f"cases[{index}].confusion_family must be disjoint from qrels")
|
|
elif len({catalog_metadata[item].get("product_id") for item in confusion}) != 1:
|
|
problems.append(f"cases[{index}].confusion_family must belong to one product")
|
|
else:
|
|
sibling_confusion = True
|
|
judged_products = {str(catalog_metadata[item].get("product_id", "")) for item in judged}
|
|
if judged_products == catalog_products:
|
|
coverage.add("all_reviewed_products")
|
|
if {"read", "write", "destructive"}.issubset(effects):
|
|
coverage.add("read_write_destructive")
|
|
if sibling_confusion:
|
|
coverage.add("sibling_confusion")
|
|
if graded_equivalent:
|
|
coverage.add("graded_equivalent_entrypoints")
|
|
if forbidden_alternative:
|
|
coverage.add("forbidden_with_alternative_gold")
|
|
if workflow_ok:
|
|
coverage.add("two_to_four_step_workflows")
|
|
return coverage
|
|
|
|
|
|
def validate_independent_result(
|
|
result: Any,
|
|
independent: dict[str, Any],
|
|
proxy: dict[str, Any],
|
|
counts: Counter[str],
|
|
problems: list[str],
|
|
) -> None:
|
|
if not isinstance(result, dict) or result.get("version") != "tool-search-independent-evaluation.v1":
|
|
problems.append("independent result version must be tool-search-independent-evaluation.v1")
|
|
return
|
|
catalog = result.get("catalog", {})
|
|
if catalog.get("source_hash") != proxy.get("catalog_source_hash") or catalog.get("surface_hash") != proxy.get("catalog_surface_hash"):
|
|
problems.append("independent result Catalog differs from frozen proxy generation")
|
|
thresholds = independent.get("thresholds", {})
|
|
overall = result.get("overall", {})
|
|
if overall.get("cases") != sum(counts[name] for name in ["chinese_only", "mixed_chinese_ascii", "english"]):
|
|
problems.append("independent result case count differs from sealed qrels")
|
|
minimum_overall = thresholds.get("minimum_overall_recall_at_5")
|
|
if not isinstance(minimum_overall, (int, float)) or overall.get("recall_at_5", -1) < minimum_overall:
|
|
problems.append("independent overall Recall@5 is below the frozen threshold")
|
|
control = result.get("control_overall", {})
|
|
if control.get("cases") != overall.get("cases"):
|
|
problems.append("independent control case count differs from candidate")
|
|
delta = result.get("recall_at_5_delta")
|
|
ci = result.get("product_cluster_recall_at_5_delta_ci_95", {})
|
|
noninferiority_margin = thresholds.get("recall_at_5_noninferiority_margin")
|
|
if not isinstance(delta, (int, float)) or not isinstance(ci.get("lower"), (int, float)) or not isinstance(ci.get("upper"), (int, float)):
|
|
problems.append("independent paired BM25 delta and product-cluster CI are required")
|
|
elif not isinstance(noninferiority_margin, (int, float)) or ci["lower"] < -noninferiority_margin:
|
|
problems.append("independent candidate fails BM25 Recall@5 noninferiority")
|
|
if thresholds.get("default_switch_requested"):
|
|
minimum_effect = thresholds.get("minimum_default_switch_recall_at_5_gain")
|
|
if not isinstance(minimum_effect, (int, float)) or ci.get("lower", float("-inf")) <= minimum_effect:
|
|
problems.append("independent candidate lacks the frozen Recall@5 gain for a default switch")
|
|
slices = result.get("language_slices", {})
|
|
minimum_slice = thresholds.get("minimum_language_recall_at_5")
|
|
for name in ["chinese_only", "mixed_chinese_ascii", "english"]:
|
|
payload = slices.get(name, {})
|
|
if payload.get("cases") != counts[name]:
|
|
problems.append(f"independent result {name} count differs from sealed qrels")
|
|
if not isinstance(minimum_slice, (int, float)) or payload.get("recall_at_5", -1) < minimum_slice:
|
|
problems.append(f"independent result {name} Recall@5 is below the frozen threshold")
|
|
safety = result.get("safety", {})
|
|
maximum_forbidden = thresholds.get("maximum_forbidden_exposure_at_5")
|
|
minimum_alternative = thresholds.get("minimum_alternative_recall_at_5")
|
|
maximum_sibling = thresholds.get("maximum_sibling_exposure_at_5")
|
|
if not isinstance(maximum_forbidden, (int, float)) or safety.get("forbidden_exposure_at_5", 2) > maximum_forbidden:
|
|
problems.append("independent forbidden exposure is above the frozen threshold")
|
|
if not isinstance(minimum_alternative, (int, float)) or safety.get("alternative_recall_at_5", -1) < minimum_alternative:
|
|
problems.append("independent alternative Recall@5 is below the frozen threshold")
|
|
if not isinstance(maximum_sibling, (int, float)) or safety.get("sibling_exposure_at_5", 2) > maximum_sibling:
|
|
problems.append("independent sibling exposure is above the frozen threshold")
|
|
workflow = result.get("workflow", {})
|
|
if workflow.get("cases") != counts["workflow"]:
|
|
problems.append("independent workflow count differs from sealed qrels")
|
|
if workflow.get("complete_at_5", -1) < thresholds.get("minimum_workflow_complete_at_5", 2):
|
|
problems.append("independent workflow Complete@5 is below the frozen threshold")
|
|
if workflow.get("required_recall_at_5", -1) < thresholds.get("minimum_workflow_required_recall_at_5", 2):
|
|
problems.append("independent workflow required Recall@5 is below the frozen threshold")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
parser.add_argument("--catalog-dir", type=Path, default=DEFAULT_CATALOG_DIR)
|
|
parser.add_argument("--require-sealed", action="store_true")
|
|
parser.add_argument("--independent-result", type=Path)
|
|
args = parser.parse_args()
|
|
problems = validate_manifest(
|
|
args.manifest.resolve(),
|
|
args.require_sealed,
|
|
args.catalog_dir.resolve(),
|
|
args.independent_result.resolve() if args.independent_result else None,
|
|
)
|
|
if problems:
|
|
for problem in problems:
|
|
print(f"ERROR: {problem}")
|
|
return 1
|
|
print("tool search evaluation manifest: OK")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|