diff --git a/.github/eval-allowlist.txt b/.github/eval-allowlist.txt new file mode 100644 index 00000000..8cde985f --- /dev/null +++ b/.github/eval-allowlist.txt @@ -0,0 +1,61 @@ +# /eval 自助触发允许名单 +# +# 名单内的 GitHub 登录名可对【自己创建的 PR】触发 /eval 评测； +# 对任意 PR 触发仍需仓库 write/maintain/admin 权限（维护者背书）。 +# 授权读取的始终是默认分支上的本文件，PR 无法修改自身授权。 +# +# 变更本文件必须走 PR 评审。每行一个 GitHub login，# 开头为注释。 + +aftersss +notable-open +EdgarWang0925 +ayunya +yutongshe +qingyang1014 +caiTriumph +xlb1130 +Anonymity-0 +FuShu-Yang +guimingyue +AlwaysLee +TaoJikun +zengyoulingzyl-stack +liyuan333 +huangyoo +lifeihong +nitonitori +cywan1998 +gangwn +junlonghuo2 +aqruan +Freda0909 +ShawnWhite777 +PeterGuy326 +abucraft +pengzhihan47-star +rainyak8 +gongrongyun +huangyuanzhuo-coder +ybcstudy +bigqy +liwang-ai +meng93 +wxianfeng +Patrick-Star-CN +rossluo28-hz +dxy704330469 +gtezg30062 +Neige-Premaire +zhuoyu20 +avicii-chen +typefield +Haofeng0705 +Huwenjiao +liuzeyang +maoqxxmm +FloralTide +lingyun9833 +dxb121 +C0922 +xiaoji121 +H3java diff --git a/.github/workflows/eval-dispatch.yml b/.github/workflows/eval-dispatch.yml new file mode 100644 index 00000000..70df32e4 --- /dev/null +++ b/.github/workflows/eval-dispatch.yml @@ -0,0 +1,139 @@ +name: PR Eval Dispatch + +# `/eval [sha=] [cases=]` PR 评论 → 触发内网评测流水线，报告由内网 bot 回贴。 +# 本 workflow 只在默认分支上下文运行，不 checkout、不执行 PR 代码。 +# 审核 SHA 规则：评测他人 PR 必须显式携带 sha=（审阅背书凭据，验证 +# 其恰为当前 open head）；评测自己创建的 PR 可省略，自动钉住派发时刻 +# 的当前 head（作者自背书，无第三方偷换窗口）；内网 CI 另以 +# FETCH_HEAD 校验兜底派发后的变更。 +# 授权两级：仓库 write/maintain/admin 可派发任意 PR；默认分支 +# .github/eval-allowlist.txt 名单内的用户仅可派发自己创建的 PR。 +# 触发通道与凭证全部经 secrets 注入，文件内不出现任何内网信息。 + +on: + issue_comment: + types: + - created + +permissions: {} + +concurrency: + group: eval-dispatch-${{ github.event.issue.number }} + cancel-in-progress: false + +jobs: + dispatch: + name: Dispatch internal evaluation + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/eval') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + pull-requests: read + steps: + - name: Check out default branch tooling + uses: actions/checkout@v4 + + - name: Verify commenter dispatch authorization + env: + GH_TOKEN: ${{ github.token }} + COMMENTER: ${{ github.event.comment.user.login }} + PR_AUTHOR: ${{ github.event.issue.user.login }} + EVAL_ALLOWLIST_PATH: .github/eval-allowlist.txt + run: | + # 不用 --fail：非协作者查权限返回 404 错误体，交由 guard 走名单分支；硬网络错误降级为空对象同样 fail-closed + permission_json="$(curl --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${COMMENTER}/permission")" || permission_json='{}' + printf '%s' "$permission_json" | python3 scripts/ci/eval_dispatch_guard.py permission + + - name: Parse /eval command + id: parse + continue-on-error: true + env: + COMMENT_BODY: ${{ github.event.comment.body }} + run: python3 scripts/ci/eval_comment_parse.py + + - name: Reply usage on parse failure + if: steps.parse.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + PARSE_ERROR: ${{ steps.parse.outputs.error }} + run: | + body="❌ /eval 命令解析失败：${PARSE_ERROR}" + jq -n --arg body "$body" '{body: $body}' | curl --fail --silent --show-error \ + -X POST \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + --data @- \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null + exit 1 + + - name: Verify reviewed PR head + id: pr + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + EXPECTED_PR_NUMBER: ${{ github.event.issue.number }} + REVIEWED_SHA: ${{ steps.parse.outputs.reviewed_sha }} + COMMENTER: ${{ github.event.comment.user.login }} + run: | + pr_json="$(curl --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + printf '%s' "$pr_json" \ + | python3 scripts/ci/eval_dispatch_guard.py head \ + >> "$GITHUB_OUTPUT" + + - name: Trigger internal evaluation pipeline + env: + EVAL_TRIGGER_TOKEN: ${{ secrets.EVAL_TRIGGER_TOKEN }} + EVAL_TRIGGER_URL: ${{ secrets.EVAL_TRIGGER_URL }} + PR_NUMBER: ${{ github.event.issue.number }} + PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + PRODUCTS: ${{ steps.parse.outputs.products }} + CASES_REF: ${{ steps.parse.outputs.cases_ref }} + run: | + if [ -z "$EVAL_TRIGGER_TOKEN" ] || [ -z "$EVAL_TRIGGER_URL" ]; then + echo "EVAL_TRIGGER_URL / EVAL_TRIGGER_TOKEN not configured; cannot dispatch." >&2 + exit 1 + fi + jq -n \ + --arg pr "$PR_NUMBER" \ + --arg sha "$PR_HEAD_SHA" \ + --arg products "$PRODUCTS" \ + --arg cases "$CASES_REF" \ + '{branch: "main", params: {pr_number: $pr, pr_head_sha: $sha, products: $products, cases_ref: $cases}}' \ + | curl --fail --silent --show-error \ + -X POST \ + -H "private-token: ${EVAL_TRIGGER_TOKEN}" \ + -H "Content-Type: application/json" \ + --data @- \ + "$EVAL_TRIGGER_URL" + echo "Internal evaluation dispatched." + + - name: Acknowledge on PR + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + PRODUCTS: ${{ steps.parse.outputs.products }} + CASES_REF: ${{ steps.parse.outputs.cases_ref }} + run: | + cases_note="" + if [ -n "$CASES_REF" ]; then + cases_note="，用例版本 \`${CASES_REF}\`" + fi + body="🛰️ /eval 已受理：产品集 \`${PRODUCTS}\`${cases_note}，评测对象 \`${PR_HEAD_SHA}\`。内网评测流水线运行结束后将由 bot 回贴报告（首行为基线对比头条）。" + jq -n --arg body "$body" '{body: $body}' | curl --fail --silent --show-error \ + -X POST \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + --data @- \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null diff --git a/scripts/ci/eval_comment_parse.py b/scripts/ci/eval_comment_parse.py new file mode 100644 index 00000000..24596151 --- /dev/null +++ b/scripts/ci/eval_comment_parse.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""解析 PR 评论中的 `/eval [sha=] [cases=]` 触发命令。 + +供 .github/workflows/eval-dispatch.yml 调用：评论体经环境变量 COMMENT_BODY +传入（避免 argv 注入），解析结果以 GitHub Actions output +（products / cases_ref / reviewed_sha）写出；格式非法时把单行错误写入 +output `error` 并以非零退出。 + +产品集显式必填；`sha=` 在评测他人 PR 时必填（审核背书凭据，由 guard 校验）， +评测自己创建的 PR 时可省略（自助模式，自动钉住派发时刻的当前 head）； +`cases=` 为用例仓库版本逃生舱（破坏性变更配对验证）。 +""" + +import os +import re +import sys + +PRODUCT_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +REF_CHARSET_RE = re.compile(r"^[A-Za-z0-9._/-]+$") +SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") +USAGE = ( + "用法: /eval [,...] [sha=<40位PR-head-SHA>] [cases=]；" + "评测他人 PR 时 sha= 必填，自己的 PR 可省略；" + "示例: /eval drive,doc sha=0123456789abcdef0123456789abcdef01234567" +) + + +def _is_valid_cases_ref(ref: str) -> bool: + """等价于 git check-ref-format --allow-onelevel 的结构校验（不执行 git）。 + + 字符白名单之上补齐结构规则：禁止首尾斜杠与连续斜杠、尾部点号、 + 任意位置的 `..`、以点开头或以 .lock 结尾的路径分量；另禁止 `-` + 开头，避免被下游 `git fetch origin ` 当作选项解析。 + """ + if not ref or not REF_CHARSET_RE.match(ref): + return False + if ref.startswith(("-", "/")) or ref.endswith(("/", ".")): + return False + if ".." in ref or "//" in ref: + return False + for component in ref.split("/"): + if component.startswith(".") or component.endswith(".lock"): + return False + return True + + +def parse(body: str): + lines = [line.strip() for line in (body or "").splitlines() if line.strip()] + if not lines: + raise ValueError(f"评论为空。{USAGE}") + tokens = lines[0].split() + if tokens[0] != "/eval": + raise ValueError(f"首行必须以 /eval 命令开头。{USAGE}") + if len(tokens) < 2: + raise ValueError(f"产品集显式必填。{USAGE}") + + products = [p for p in tokens[1].split(",") if p] + if not products: + raise ValueError(f"产品集显式必填。{USAGE}") + for product in products: + if not PRODUCT_RE.match(product): + raise ValueError(f"产品名非法: {product}。{USAGE}") + + cases_ref = "" + reviewed_sha = "" + for extra in tokens[2:]: + if extra.startswith("cases="): + if cases_ref: + raise ValueError(f"cases 引用只能指定一次。{USAGE}") + cases_ref = extra[len("cases="):] + if not _is_valid_cases_ref(cases_ref): + raise ValueError(f"cases 引用非法: {extra}。{USAGE}") + elif extra.startswith("sha="): + if reviewed_sha: + raise ValueError(f"审核 SHA 只能指定一次。{USAGE}") + reviewed_sha = extra[len("sha="):] + if not SHA_RE.match(reviewed_sha): + raise ValueError(f"审核 SHA 非法: {extra}。{USAGE}") + else: + raise ValueError(f"未知参数: {extra}。{USAGE}") + + if not reviewed_sha: + # 是否允许省略由 guard 按“评论者是否 PR 作者”裁决，解析层不拦 + return ",".join(products), cases_ref, "" + + return ",".join(products), cases_ref, reviewed_sha.lower() + + +def _write_outputs(pairs): + lines = [f"{key}={value}" for key, value in pairs] + output_path = os.environ.get("GITHUB_OUTPUT", "") + if output_path: + with open(output_path, "a", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + print("\n".join(lines)) + + +def main() -> int: + try: + products, cases_ref, reviewed_sha = parse(os.environ.get("COMMENT_BODY", "")) + except ValueError as exc: + _write_outputs([("error", str(exc))]) + print(str(exc), file=sys.stderr) + return 1 + _write_outputs( + [ + ("products", products), + ("cases_ref", cases_ref), + ("reviewed_sha", reviewed_sha), + ] + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/eval_comment_parse_test.py b/scripts/ci/eval_comment_parse_test.py new file mode 100644 index 00000000..419b1f7c --- /dev/null +++ b/scripts/ci/eval_comment_parse_test.py @@ -0,0 +1,92 @@ +import pytest + +from eval_comment_parse import parse + +SHA = "0123456789abcdef0123456789abcdef01234567" + + +def test_single_product(): + assert parse(f"/eval drive sha={SHA}") == ("drive", "", SHA) + + +def test_multiple_products(): + assert parse(f"/eval drive,doc,edu-app sha={SHA}") == ("drive,doc,edu-app", "", SHA) + + +def test_cases_ref_escape_hatch(): + assert parse(f"/eval drive sha={SHA} cases=feat/drive-latest") == ( + "drive", + "feat/drive-latest", + SHA, + ) + + +def test_extra_lines_after_command_are_ignored(): + body = f"/eval drive sha={SHA}\n\n顺便说明：这个 PR 只动了 drive 的 --latest。" + assert parse(body) == ("drive", "", SHA) + + +def test_missing_products_rejected(): + with pytest.raises(ValueError, match="产品集显式必填"): + parse("/eval") + + +def test_omitted_reviewed_sha_defers_to_guard(): + # 是否允许省略由 guard 按“评论者是否 PR 作者”裁决，解析层放行并输出空 reviewed_sha + assert parse("/eval drive") == ("drive", "", "") + + +def test_similar_command_prefix_rejected(): + with pytest.raises(ValueError, match="/eval 命令开头"): + parse(f"/evaluate drive sha={SHA}") + + +def test_illegal_product_name_rejected(): + with pytest.raises(ValueError, match="产品名非法"): + parse(f"/eval drive;rm sha={SHA}") + + +def test_unknown_extra_token_rejected(): + with pytest.raises(ValueError, match="未知参数"): + parse(f"/eval drive sha={SHA} --force") + + +def test_illegal_cases_ref_rejected(): + with pytest.raises(ValueError, match="cases 引用非法"): + parse(f"/eval drive sha={SHA} cases=$(whoami)") + + +def test_structurally_invalid_cases_refs_rejected(): + # git check-ref-format 结构规则：字符合法但结构非法的引用必须拒绝 + invalid_refs = [ + "..", # 以点开头且含连续点号 + "/main", # 首部斜杠 + "feature//x", # 连续斜杠 + "feature/", # 尾部斜杠 + "release.", # 尾部点号 + "a..b", # 任意位置连续点号 + ".hidden", # 分量以点开头 + "a/.hidden", # 非首分量以点开头 + "a.lock", # 分量以 .lock 结尾 + "a/b.lock", # 非首分量以 .lock 结尾 + "-flag", # 以 - 开头，防 git fetch 选项注入 + ] + for ref in invalid_refs: + with pytest.raises(ValueError, match="cases 引用非法"): + parse(f"/eval drive sha={SHA} cases={ref}") + + +def test_structurally_valid_cases_refs_accepted(): + valid_refs = ["main", "feat/drive-latest", "release/v1.0", "a.b/c-d_e", SHA] + for ref in valid_refs: + assert parse(f"/eval drive sha={SHA} cases={ref}") == ("drive", ref, SHA) + + +def test_short_reviewed_sha_rejected(): + with pytest.raises(ValueError, match="审核 SHA 非法"): + parse("/eval drive sha=0123456") + + +def test_empty_comment_rejected(): + with pytest.raises(ValueError, match="评论为空"): + parse(" \n ") diff --git a/scripts/ci/eval_dispatch_guard.py b/scripts/ci/eval_dispatch_guard.py new file mode 100644 index 00000000..d4be187a --- /dev/null +++ b/scripts/ci/eval_dispatch_guard.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""校验 `/eval` 派发的仓库权限与已审核 PR head。""" + +import json +import os +import re +import sys + + +TRUSTED_PERMISSIONS = frozenset({"write", "maintain", "admin"}) +FULL_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") + + +def _read_api_response(): + try: + value = json.load(sys.stdin) + except (json.JSONDecodeError, OSError) as exc: + raise ValueError(f"GitHub API response is not valid JSON: {exc}") from exc + if not isinstance(value, dict): + raise ValueError("GitHub API response must be a JSON object") + return value + + +def load_allowlist(path: str) -> frozenset: + """读取自助触发允许名单（默认分支文件；# 注释与空行忽略；大小写不敏感）。""" + if not path: + return frozenset() + try: + with open(path, encoding="utf-8") as f: + lines = f.read().splitlines() + except FileNotFoundError: + return frozenset() + entries = set() + for line in lines: + entry = line.split("#", 1)[0].strip() + if entry: + entries.add(entry.lower()) + return frozenset(entries) + + +def authorize_dispatch(response, commenter: str, pr_author: str, allowlist: frozenset): + """两级授权：仓库写权限可派发任意 PR；名单内用户仅可派发自己创建的 PR。""" + permission = response.get("permission") + if permission in TRUSTED_PERMISSIONS: + return f"maintainer:{permission}" + commenter_key = (commenter or "").strip().lower() + author_key = (pr_author or "").strip().lower() + if commenter_key and commenter_key in allowlist: + if commenter_key == author_key: + return "allowlisted-author" + raise PermissionError( + f"{commenter or 'commenter'} is allowlisted for self-service only " + "and may not dispatch other authors' PRs" + ) + raise PermissionError( + f"{commenter or 'commenter'} does not have write, maintain, or admin permission " + "and is not an allowlisted PR author" + ) + + +def require_reviewed_current_head(response, expected_pr_number: str, reviewed_sha: str, commenter: str): + try: + expected_number = int(expected_pr_number) + except (TypeError, ValueError) as exc: + raise ValueError("expected PR number is invalid") from exc + if response.get("number") != expected_number: + raise ValueError("GitHub API response does not match the requested PR") + if response.get("state") != "open": + raise ValueError("PR is not open") + + head = response.get("head") + current_sha = head.get("sha") if isinstance(head, dict) else "" + if not FULL_SHA_RE.fullmatch(current_sha or ""): + raise ValueError("GitHub API response does not contain a valid PR head SHA") + + if reviewed_sha: + if not FULL_SHA_RE.fullmatch(reviewed_sha): + raise ValueError("reviewed SHA must be a full 40-character commit SHA") + if current_sha.lower() != reviewed_sha.lower(): + raise ValueError( + f"PR head changed after review: reviewed {reviewed_sha.lower()}, " + f"current {current_sha.lower()}" + ) + return reviewed_sha.lower() + + # sha 省略：仅限评论者本人创建的 PR（自助模式）——作者对自己分支的 + # tip 背书不存在第三方偷换窗口；钉住派发时刻的当前 head，内网侧 + # FETCH_HEAD 校验继续兜底派发→执行窗口。评测他人 PR 必须显式带 sha=。 + author = ((response.get("user") or {}).get("login") or "").strip().lower() + commenter_key = (commenter or "").strip().lower() + if not author or commenter_key != author: + raise ValueError( + "dispatching another author's PR requires an explicit reviewed sha=" + ) + return current_sha.lower() + + +def main() -> int: + mode = sys.argv[1] if len(sys.argv) == 2 else "" + try: + response = _read_api_response() + if mode == "permission": + role = authorize_dispatch( + response, + os.environ.get("COMMENTER", ""), + os.environ.get("PR_AUTHOR", ""), + load_allowlist(os.environ.get("EVAL_ALLOWLIST_PATH", "")), + ) + print(f"authorized={role}") + return 0 + if mode == "head": + head_sha = require_reviewed_current_head( + response, + os.environ.get("EXPECTED_PR_NUMBER", ""), + os.environ.get("REVIEWED_SHA", ""), + os.environ.get("COMMENTER", ""), + ) + print(f"head_sha={head_sha}") + return 0 + raise ValueError(f"unknown guard mode: {mode}") + except (PermissionError, ValueError) as exc: + print(str(exc), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/scripts/eval_dispatch_workflow_test.go b/test/scripts/eval_dispatch_workflow_test.go new file mode 100644 index 00000000..235d1f31 --- /dev/null +++ b/test/scripts/eval_dispatch_workflow_test.go @@ -0,0 +1,291 @@ +// Copyright 2026 Alibaba Group +// Licensed under the Apache License, Version 2.0 (the "License"); + +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestEvalDispatchWorkflowUsesRepositoryPermissionAndReviewedSHA(t *testing.T) { + t.Parallel() + + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("resolve repository root: %v", err) + } + data, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "eval-dispatch.yml")) + if err != nil { + t.Fatalf("read eval-dispatch workflow: %v", err) + } + workflow := string(data) + + for _, want := range []string{ + "/collaborators/${COMMENTER}/permission", + "eval_dispatch_guard.py permission", + "PR_AUTHOR: ${{ github.event.issue.user.login }}", + "EVAL_ALLOWLIST_PATH: .github/eval-allowlist.txt", + "REVIEWED_SHA: ${{ steps.parse.outputs.reviewed_sha }}", + "eval_dispatch_guard.py head", + } { + if !strings.Contains(workflow, want) { + t.Errorf("eval-dispatch workflow missing security contract %q", want) + } + } + if strings.Contains(workflow, "author_association") { + t.Error("eval-dispatch workflow must not authorize from author_association") + } + if _, err := os.Stat(filepath.Join(root, ".github", "eval-allowlist.txt")); err != nil { + t.Errorf("eval allowlist file missing on default branch: %v", err) + } +} + +func TestEvalDispatchRejectsLowRepositoryPermissions(t *testing.T) { + t.Parallel() + + for _, permission := range []string{"read", "triage", "none"} { + permission := permission + t.Run(permission, func(t *testing.T) { + t.Parallel() + output, err := runEvalDispatchGuard(t, "permission", `{"permission":"`+permission+`"}`, "COMMENTER=low-privilege-user") + if err == nil { + t.Fatalf("permission %q unexpectedly allowed; output=%s", permission, output) + } + if !strings.Contains(output, "does not have write, maintain, or admin permission") { + t.Fatalf("permission %q rejection = %q, want trusted-permission error", permission, output) + } + }) + } +} + +func TestEvalDispatchAllowsTrustedRepositoryPermissions(t *testing.T) { + t.Parallel() + + for _, permission := range []string{"write", "maintain", "admin"} { + permission := permission + t.Run(permission, func(t *testing.T) { + t.Parallel() + output, err := runEvalDispatchGuard(t, "permission", `{"permission":"`+permission+`"}`, "COMMENTER=maintainer") + if err != nil { + t.Fatalf("permission %q rejected: %v\n%s", permission, err, output) + } + }) + } +} + +func TestEvalDispatchRejectsChangedPRHead(t *testing.T) { + t.Parallel() + + const reviewedSHA = "1111111111111111111111111111111111111111" + const currentSHA = "2222222222222222222222222222222222222222" + pr := `{"number":934,"state":"open","head":{"sha":"` + currentSHA + `"}}` + output, err := runEvalDispatchGuard( + t, + "head", + pr, + "EXPECTED_PR_NUMBER=934", + "REVIEWED_SHA="+reviewedSHA, + ) + if err == nil { + t.Fatalf("changed PR head unexpectedly allowed; output=%s", output) + } + if !strings.Contains(output, "PR head changed after review") { + t.Fatalf("changed-head rejection = %q, want explicit stale-review error", output) + } +} + +func TestEvalDispatchAcceptsReviewedCurrentPRHead(t *testing.T) { + t.Parallel() + + const reviewedSHA = "1111111111111111111111111111111111111111" + pr := `{"number":934,"state":"open","head":{"sha":"` + reviewedSHA + `"}}` + output, err := runEvalDispatchGuard( + t, + "head", + pr, + "EXPECTED_PR_NUMBER=934", + "REVIEWED_SHA="+reviewedSHA, + ) + if err != nil { + t.Fatalf("reviewed current PR head rejected: %v\n%s", err, output) + } + if !strings.Contains(output, "head_sha="+reviewedSHA) { + t.Fatalf("head guard output = %q, want pinned head SHA", output) + } +} + +func TestEvalCommentRequiresExplicitReviewedSHA(t *testing.T) { + t.Parallel() + + const reviewedSHA = "1111111111111111111111111111111111111111" + output, err := runEvalCommentParser(t, "/eval drive sha="+reviewedSHA) + if err != nil { + t.Fatalf("explicit reviewed SHA rejected: %v\n%s", err, output) + } + for _, want := range []string{"products=drive", "reviewed_sha=" + reviewedSHA} { + if !strings.Contains(output, want) { + t.Errorf("parser output = %q, want %q", output, want) + } + } + + // sha 省略在解析层放行（空 reviewed_sha），由 guard 按评论者是否 PR 作者裁决 + output, err = runEvalCommentParser(t, "/eval drive") + if err != nil { + t.Fatalf("omitted reviewed SHA rejected at parse layer: %v\n%s", err, output) + } + if !strings.Contains(output, "reviewed_sha=\n") && !strings.HasSuffix(output, "reviewed_sha=") { + t.Fatalf("parser output = %q, want empty reviewed_sha passthrough", output) + } +} + +func TestEvalDispatchAutoPinsOwnPRHeadWhenShaOmitted(t *testing.T) { + t.Parallel() + + const currentSHA = "3333333333333333333333333333333333333333" + pr := `{"number":934,"state":"open","user":{"login":"Internal-Contributor"},"head":{"sha":"` + currentSHA + `"}}` + output, err := runEvalDispatchGuard( + t, + "head", + pr, + "EXPECTED_PR_NUMBER=934", + "REVIEWED_SHA=", + "COMMENTER=internal-contributor", + ) + if err != nil { + t.Fatalf("own-PR auto pin rejected: %v\n%s", err, output) + } + if !strings.Contains(output, "head_sha="+currentSHA) { + t.Fatalf("guard output = %q, want auto-pinned current head", output) + } +} + +func TestEvalDispatchRequiresShaForOthersPRs(t *testing.T) { + t.Parallel() + + const currentSHA = "3333333333333333333333333333333333333333" + pr := `{"number":934,"state":"open","user":{"login":"someone-else"},"head":{"sha":"` + currentSHA + `"}}` + output, err := runEvalDispatchGuard( + t, + "head", + pr, + "EXPECTED_PR_NUMBER=934", + "REVIEWED_SHA=", + "COMMENTER=internal-contributor", + ) + if err == nil { + t.Fatalf("other author's PR without sha unexpectedly allowed; output=%s", output) + } + if !strings.Contains(output, "requires an explicit reviewed sha") { + t.Fatalf("missing-sha rejection = %q, want explicit-sha requirement", output) + } +} + +func TestEvalDispatchAllowsAllowlistedAuthorOnOwnPR(t *testing.T) { + t.Parallel() + + allowlist := writeEvalAllowlist(t, "# 自助评测名单\nInternal-Contributor # 内部贡献者\n") + output, err := runEvalDispatchGuard( + t, + "permission", + `{"message":"Not Found"}`, + "COMMENTER=internal-contributor", + "PR_AUTHOR=Internal-Contributor", + "EVAL_ALLOWLIST_PATH="+allowlist, + ) + if err != nil { + t.Fatalf("allowlisted author rejected on own PR: %v\n%s", err, output) + } + if !strings.Contains(output, "authorized=allowlisted-author") { + t.Fatalf("guard output = %q, want allowlisted-author authorization", output) + } +} + +func TestEvalDispatchRejectsAllowlistedUserOnOthersPR(t *testing.T) { + t.Parallel() + + allowlist := writeEvalAllowlist(t, "internal-contributor\n") + output, err := runEvalDispatchGuard( + t, + "permission", + `{"message":"Not Found"}`, + "COMMENTER=internal-contributor", + "PR_AUTHOR=someone-else", + "EVAL_ALLOWLIST_PATH="+allowlist, + ) + if err == nil { + t.Fatalf("allowlisted user unexpectedly dispatched another author's PR; output=%s", output) + } + if !strings.Contains(output, "self-service only") { + t.Fatalf("cross-PR rejection = %q, want self-service-only error", output) + } +} + +func TestEvalDispatchRejectsUnlistedCommenterWithoutWrite(t *testing.T) { + t.Parallel() + + allowlist := writeEvalAllowlist(t, "# 空名单\n") + output, err := runEvalDispatchGuard( + t, + "permission", + `{"permission":"read"}`, + "COMMENTER=stranger", + "PR_AUTHOR=stranger", + "EVAL_ALLOWLIST_PATH="+allowlist, + ) + if err == nil { + t.Fatalf("unlisted commenter unexpectedly allowed; output=%s", output) + } + if !strings.Contains(output, "not an allowlisted PR author") { + t.Fatalf("unlisted rejection = %q, want allowlist-aware error", output) + } +} + +func writeEvalAllowlist(t *testing.T, content string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "eval-allowlist.txt") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write allowlist fixture: %v", err) + } + return path +} + +func runEvalDispatchGuard(t *testing.T, mode, input string, env ...string) (string, error) { + t.Helper() + + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("resolve repository root: %v", err) + } + cmd := exec.Command("python3", filepath.Join(root, "scripts", "ci", "eval_dispatch_guard.py"), mode) + cmd.Stdin = strings.NewReader(input) + cmd.Env = append(os.Environ(), env...) + output, runErr := cmd.CombinedOutput() + return string(output), runErr +} + +func runEvalCommentParser(t *testing.T, comment string) (string, error) { + t.Helper() + + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("resolve repository root: %v", err) + } + outputPath := filepath.Join(t.TempDir(), "github-output") + cmd := exec.Command("python3", filepath.Join(root, "scripts", "ci", "eval_comment_parse.py")) + cmd.Env = append( + os.Environ(), + "COMMENT_BODY="+comment, + "GITHUB_OUTPUT="+outputPath, + ) + combined, runErr := cmd.CombinedOutput() + fileOutput, readErr := os.ReadFile(outputPath) + if readErr != nil && !os.IsNotExist(readErr) { + t.Fatalf("read parser GitHub output: %v", readErr) + } + return string(combined) + string(fileOutput), runErr +}