feat(ci): PR 评论 /eval 触发内网 MCP 评测的 dispatch workflow
- issue_comment 触发，author_association ∈ OWNER/MEMBER/COLLABORATOR 门控 - 不 checkout、不执行 PR 代码；触发通道与凭证全部经 secrets 注入 - scripts/ci/eval_comment_parse.py 解析 /eval <products> [cases=<ref>]（10 个单测）
This commit is contained in:
chichuan
@@ -0,0 +1,117 @@
|
||||
name: PR Eval Dispatch
|
||||
|
||||
# `/eval <products> [cases=<ref>]` PR 评论 → 触发内网评测流水线，报告由内网 bot 回贴。
|
||||
# 本 workflow 只在默认分支上下文运行，不 checkout、不执行 PR 代码；
|
||||
# maintainer 打出 /eval 即代表已审阅 PR 代码（内网 CI 将构建并执行该 head）。
|
||||
# 触发通道与凭证全部经 secrets 注入，文件内不出现任何内网信息。
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types:
|
||||
- created
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: eval-dispatch-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
name: Dispatch internal evaluation
|
||||
if: >-
|
||||
github.event.issue.pull_request &&
|
||||
startsWith(github.event.comment.body, '/eval') &&
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Check out default branch tooling
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Parse /eval command
|
||||
id: parse
|
||||
continue-on-error: true
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: python3 scripts/ci/eval_comment_parse.py
|
||||
|
||||
- name: Reply usage on parse failure
|
||||
if: steps.parse.outcome == 'failure'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
PARSE_ERROR: ${{ steps.parse.outputs.error }}
|
||||
run: |
|
||||
body="❌ /eval 命令解析失败：${PARSE_ERROR}"
|
||||
jq -n --arg body "$body" '{body: $body}' | curl --fail --silent --show-error \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${GH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
--data @- \
|
||||
"https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null
|
||||
exit 1
|
||||
|
||||
- name: Resolve PR head SHA
|
||||
id: pr
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
head_sha="$(curl --fail --silent --show-error \
|
||||
-H "Authorization: Bearer ${GH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" | jq -r '.head.sha')"
|
||||
if [ -z "$head_sha" ] || [ "$head_sha" = "null" ]; then
|
||||
echo "Failed to resolve PR head SHA." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "head_sha=${head_sha}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Trigger internal evaluation pipeline
|
||||
env:
|
||||
EVAL_TRIGGER_TOKEN: ${{ secrets.EVAL_TRIGGER_TOKEN }}
|
||||
EVAL_TRIGGER_URL: ${{ secrets.EVAL_TRIGGER_URL }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }}
|
||||
PRODUCTS: ${{ steps.parse.outputs.products }}
|
||||
CASES_REF: ${{ steps.parse.outputs.cases_ref }}
|
||||
run: |
|
||||
if [ -z "$EVAL_TRIGGER_TOKEN" ] || [ -z "$EVAL_TRIGGER_URL" ]; then
|
||||
echo "EVAL_TRIGGER_URL / EVAL_TRIGGER_TOKEN not configured; cannot dispatch." >&2
|
||||
exit 1
|
||||
fi
|
||||
curl --fail --silent --show-error \
|
||||
-X POST \
|
||||
-F "token=${EVAL_TRIGGER_TOKEN}" \
|
||||
-F "ref=main" \
|
||||
-F "variables[PR_NUMBER]=${PR_NUMBER}" \
|
||||
-F "variables[PR_HEAD_SHA]=${PR_HEAD_SHA}" \
|
||||
-F "variables[PRODUCTS]=${PRODUCTS}" \
|
||||
-F "variables[CASES_REF]=${CASES_REF}" \
|
||||
"$EVAL_TRIGGER_URL"
|
||||
echo "Internal evaluation dispatched."
|
||||
|
||||
- name: Acknowledge on PR
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }}
|
||||
PRODUCTS: ${{ steps.parse.outputs.products }}
|
||||
CASES_REF: ${{ steps.parse.outputs.cases_ref }}
|
||||
run: |
|
||||
cases_note=""
|
||||
if [ -n "$CASES_REF" ]; then
|
||||
cases_note="，用例版本 \`${CASES_REF}\`"
|
||||
fi
|
||||
body="🛰️ /eval 已受理：产品集 \`${PRODUCTS}\`${cases_note}，评测对象 \`${PR_HEAD_SHA}\`。内网评测流水线运行结束后将由 bot 回贴报告（首行为基线对比头条）。"
|
||||
jq -n --arg body "$body" '{body: $body}' | curl --fail --silent --show-error \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${GH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
--data @- \
|
||||
"https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
"""解析 PR 评论中的 `/eval <products> [cases=<ref>]` 触发命令。
|
||||
|
||||
供 .github/workflows/eval-dispatch.yml 调用：评论体经环境变量 COMMENT_BODY
|
||||
传入（避免 argv 注入），解析结果以 GitHub Actions output（products / cases_ref）
|
||||
写出；格式非法时把单行错误写入 output `error` 并以非零退出。
|
||||
|
||||
产品集显式必填；`cases=<ref>` 为用例仓库版本逃生舱（破坏性变更配对验证）。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
PRODUCT_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
||||
REF_RE = re.compile(r"^[A-Za-z0-9._/-]+$")
|
||||
USAGE = "用法: /eval <product>[,<product>...] [cases=<ref>]；示例: /eval drive,doc cases=feat/x"
|
||||
|
||||
|
||||
def parse(body: str):
|
||||
lines = [line.strip() for line in (body or "").splitlines() if line.strip()]
|
||||
if not lines:
|
||||
raise ValueError(f"评论为空。{USAGE}")
|
||||
tokens = lines[0].split()
|
||||
if tokens[0] != "/eval":
|
||||
raise ValueError(f"首行必须以 /eval 命令开头。{USAGE}")
|
||||
if len(tokens) < 2:
|
||||
raise ValueError(f"产品集显式必填。{USAGE}")
|
||||
|
||||
products = [p for p in tokens[1].split(",") if p]
|
||||
if not products:
|
||||
raise ValueError(f"产品集显式必填。{USAGE}")
|
||||
for product in products:
|
||||
if not PRODUCT_RE.match(product):
|
||||
raise ValueError(f"产品名非法: {product}。{USAGE}")
|
||||
|
||||
cases_ref = ""
|
||||
for extra in tokens[2:]:
|
||||
if extra.startswith("cases="):
|
||||
cases_ref = extra[len("cases="):]
|
||||
if not cases_ref or not REF_RE.match(cases_ref):
|
||||
raise ValueError(f"cases 引用非法: {extra}。{USAGE}")
|
||||
else:
|
||||
raise ValueError(f"未知参数: {extra}。{USAGE}")
|
||||
|
||||
return ",".join(products), cases_ref
|
||||
|
||||
|
||||
def _write_outputs(pairs):
|
||||
lines = [f"{key}={value}" for key, value in pairs]
|
||||
output_path = os.environ.get("GITHUB_OUTPUT", "")
|
||||
if output_path:
|
||||
with open(output_path, "a", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines) + "\n")
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
products, cases_ref = parse(os.environ.get("COMMENT_BODY", ""))
|
||||
except ValueError as exc:
|
||||
_write_outputs([("error", str(exc))])
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
_write_outputs([("products", products), ("cases_ref", cases_ref)])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
import pytest
|
||||
|
||||
from eval_comment_parse import parse
|
||||
|
||||
|
||||
def test_single_product():
|
||||
assert parse("/eval drive") == ("drive", "")
|
||||
|
||||
|
||||
def test_multiple_products():
|
||||
assert parse("/eval drive,doc,edu-app") == ("drive,doc,edu-app", "")
|
||||
|
||||
|
||||
def test_cases_ref_escape_hatch():
|
||||
assert parse("/eval drive cases=feat/drive-latest") == ("drive", "feat/drive-latest")
|
||||
|
||||
|
||||
def test_extra_lines_after_command_are_ignored():
|
||||
body = "/eval drive\n\n顺便说明：这个 PR 只动了 drive 的 --latest。"
|
||||
assert parse(body) == ("drive", "")
|
||||
|
||||
|
||||
def test_missing_products_rejected():
|
||||
with pytest.raises(ValueError, match="产品集显式必填"):
|
||||
parse("/eval")
|
||||
|
||||
|
||||
def test_similar_command_prefix_rejected():
|
||||
with pytest.raises(ValueError, match="/eval 命令开头"):
|
||||
parse("/evaluate drive")
|
||||
|
||||
|
||||
def test_illegal_product_name_rejected():
|
||||
with pytest.raises(ValueError, match="产品名非法"):
|
||||
parse("/eval drive;rm")
|
||||
|
||||
|
||||
def test_unknown_extra_token_rejected():
|
||||
with pytest.raises(ValueError, match="未知参数"):
|
||||
parse("/eval drive --force")
|
||||
|
||||
|
||||
def test_illegal_cases_ref_rejected():
|
||||
with pytest.raises(ValueError, match="cases 引用非法"):
|
||||
parse("/eval drive cases=$(whoami)")
|
||||
|
||||
|
||||
def test_empty_comment_rejected():
|
||||
with pytest.raises(ValueError, match="评论为空"):
|
||||
parse(" \n ")
|
||||
Reference in New Issue
Block a user