Compare commits

...
Author SHA1 Message Date
修雨 fdb461155f Merge branch 'main' into bugfix/shortcut-fixes 2026-07-23 00:08:11 +08:00
修雨 7e38f9fbba test(chat): cover missing download message id 2026-07-22 23:57:16 +08:00
修雨 067b3e6e51 test(chat): cover download media aliases 2026-07-22 23:45:37 +08:00
修雨 bcac84987f fix(chat): preserve message and schema contracts 2026-07-22 23:34:35 +08:00
DennisandClaude Opus 4.8 8dd87108f1 fix(chat): make message-read shortcuts render cards, forwards & ciphertext
The message-list shortcuts projected raw message content, so several message
kinds came out unreadable:

- speaker showed as null (real key is the bare "sender"; forwardMessages carry
  the literal string "null")
- out-of-office auto-replies / cards dumped raw rich-content JSON
- forwarded chat records ("聊天记录") collapsed to a lossy "[卡片]" summary,
  dropping their nested forwardMessages
- encrypted card/robot messages dumped raw base64 ciphertext (dws holds no key)

Extract the projection logic into a shared internal/shortcut/chatmsg package and
fix all five projecting read commands to reuse it (+chat-messages,
+messages-list, +messages-list-direct, +at-me, +search-msg, +thread-replies),
each keeping its own output fields:

- Sender: probe the bare "sender" key first, treat "null"/"" as absent
- CleanText: extract readable text from rich-content JSON, mark ciphertext as
  [加密消息,无法解码] instead of leaking base64
- Forwarded: recursively expand forwardMessages so chat records are readable
- RecoverEncryptedContents: best-effort re-query of search_messages_by_sender
  (which returns server-rendered plaintext) matched back by openMessageId

Also add --msg-id / --open-message-id aliases to `chat message download-media`
so agents that copy the openMessageId/msgId field don't hit "unknown flag".

Verified live: +messages-list-direct expands a 44-message forwarded record;
+messages-list / +at-me render clean text; download-media accepts --msg-id.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-22 20:46:45 +08:00
修雨 412e77f215 Merge pull request #763 from DingTalk-Real-AI/codex/retry-gitee-transient-outages
fix: retry transient Gitee read outages safely
2026-07-22 17:56:50 +08:00
修雨 2a0bf1ebea fix: retry transient Gitee read outages safely 2026-07-22 17:46:03 +08:00
修雨 9ce13da6ed Merge pull request #762 from DingTalk-Real-AI/codex/extend-gitee-upload-window
fix: extend Gitee upload window
2026-07-22 16:50:49 +08:00
修雨 0e5731166b fix: extend Gitee upload window 2026-07-22 16:39:55 +08:00
修雨 92edd8ea53 Merge pull request #761 from DingTalk-Real-AI/codex/fix-gitee-slow-upload-timeout
fix: allow slow Gitee binary uploads
2026-07-22 16:08:28 +08:00
修雨 931d75beaf fix: allow slow Gitee binary uploads 2026-07-22 15:57:21 +08:00
修雨 7667cb30a3 Merge pull request #759 from DingTalk-Real-AI/codex/fix-gitee-upload-expect
fix: disable Expect for Gitee uploads
2026-07-22 15:13:33 +08:00
18 changed files with 1302 additions and 179 deletions
+13 -4
View File
@@ -1735,7 +1735,7 @@ jobs:
if: ${{ !cancelled() && vars.ENABLE_GITEE_UPLOAD_FALLBACK == 'true' && needs.release-contract.result == 'success' && needs.release.result == 'success' && needs.publish-channels.result == 'success' }}
needs: [release-contract, release, publish-channels]
runs-on: ubuntu-latest
timeout-minutes: 120
timeout-minutes: 360
permissions:
contents: read
env:
@@ -1752,12 +1752,14 @@ jobs:
- name: Check out trusted release tooling
uses: actions/checkout@v4
timeout-minutes: 5
with:
ref: ${{ github.sha }}
path: tmp/trusted-release-tooling
persist-credentials: false
- name: Fetch and verify sealed release tag
timeout-minutes: 5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
@@ -1776,7 +1778,7 @@ jobs:
path: dist
- name: Mirror release to Gitee (China)
timeout-minutes: 100
timeout-minutes: 318
run: ./scripts/release/sync-to-gitee.sh
env:
VERSION: ${{ needs.release-contract.outputs.release_version }}
@@ -2180,13 +2182,14 @@ jobs:
needs: dispatch-contract
if: ${{ !cancelled() && needs.dispatch-contract.result == 'success' && (needs.dispatch-contract.outputs.mode == 'repair_gitee' || needs.dispatch-contract.outputs.mode == 'repair_oss') && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.repository == 'DingTalk-Real-AI/dingtalk-workspace-cli' }}
runs-on: ubuntu-latest
timeout-minutes: 120
timeout-minutes: 360
permissions:
actions: read
contents: read
steps:
- name: Check out trusted release tooling
uses: actions/checkout@v4
timeout-minutes: 5
with:
ref: ${{ github.sha }}
path: tooling
@@ -2194,6 +2197,7 @@ jobs:
fetch-depth: 0
- name: Validate repair version
timeout-minutes: 2
working-directory: tooling
env:
VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }}
@@ -2208,6 +2212,7 @@ jobs:
- name: Verify immutable release authority
id: authority
uses: actions/github-script@v7
timeout-minutes: 5
env:
VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }}
with:
@@ -2330,12 +2335,14 @@ jobs:
- name: Check out sealed release source
uses: actions/checkout@v4
timeout-minutes: 5
with:
ref: ${{ steps.authority.outputs.commit_sha }}
path: release-source
persist-credentials: false
- name: Fetch and verify sealed release tag
timeout-minutes: 5
working-directory: tooling
env:
VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }}
@@ -2352,6 +2359,7 @@ jobs:
"$VERSION" "$RELEASE_COMMIT" "$RELEASE_TAG_OBJECT"
- name: Require successful Release workflow delivery
timeout-minutes: 5
working-directory: tooling
env:
VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }}
@@ -2368,6 +2376,7 @@ jobs:
--channel-repair "$target" "$VERSION" "$RELEASE_COMMIT"
- name: Download and verify immutable GitHub Release assets
timeout-minutes: 10
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ inputs.repair_gitee_version || inputs.repair_oss_version }}
@@ -2385,7 +2394,7 @@ jobs:
- name: Mirror release to Gitee (China)
if: ${{ needs.dispatch-contract.outputs.mode == 'repair_gitee' }}
timeout-minutes: 100
timeout-minutes: 318
working-directory: tooling
run: |
"$GITHUB_WORKSPACE/tooling/scripts/release/sync-to-gitee.sh"
+4
View File
@@ -62,3 +62,7 @@ dwsbin
/docs/shortcut-comparison.html
/docs/shortcut-gsb-eval.*
/scripts/run_shortcut_real_read_matrix.py
# Local coverage artifacts
coverage-shortcut.txt
coverage-*.txt
+4
View File
@@ -6,6 +6,10 @@ The format is inspired by [Keep a Changelog](https://keepachangelog.com/) and th
## [Unreleased]
### Fixed
- **Message-read shortcut projection** (#706) — the message-list shortcuts (`chat +chat-messages` / `+messages-list` / `+messages-list-direct` / `+at-me` / `+search-msg` / `+thread-replies`) now render card and out-of-office rich-content JSON as readable text (without ever rewriting ordinary text that merely embeds a JSON fragment), expand a forwarded chat record's nested `forwardMessages` instead of collapsing to a "[卡片]" summary, and mark undecryptable encrypted card messages as `[加密消息]`; the speaker is read from the bare `sender` key, nested `{name:…}` sender objects yield their display name, and the literal string `"null"` is treated as absent. Shared projection helpers now live in `internal/shortcut/chatmsg`. `chat message download-media` also gains `--msg-id` / `--open-message-id` aliases for its `--message-id` flag so agents copying the `openMessageId`/`msgId` output field no longer hit "unknown flag".
## [1.0.54] - 2026-07-21
This release promotes the validated `v1.0.54-beta.2` baseline to stable. It restores the default transport envelope for personal event output with opt-in flattening, plus Schema CLI path and plugin overlay compatibility fixes.
+26
View File
@@ -3143,6 +3143,25 @@ flow-status 取值:1=处理中(PROCESSING),2=输入中(INPUTTING),3=完成
# resource-id: 从 dws chat message list 返回的消息内容中获取 mediaId
# message-id: 从 dws chat message list 返回的 openMessageId
# open-conversation-id: 从 dws chat search 获取 openConversationId`,
PreRunE: func(cmd *cobra.Command, args []string) error {
// Cobra validates required flags after PreRunE. Copy a supplied alias
// into the canonical flag first so --message-id can remain a hard
// required fact in both the executable and Agent Schema contracts.
if cmd.Flags().Changed("message-id") {
return nil
}
alias := ""
switch {
case cmd.Flags().Changed("msg-id"):
alias = "msg-id"
case cmd.Flags().Changed("open-message-id"):
alias = "open-message-id"
default:
return nil
}
value, _ := cmd.Flags().GetString(alias) // registered string flags above
return cmd.Flags().Set("message-id", value)
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := validateRequiredFlags(cmd, "type", "resource-id", "message-id", "open-conversation-id", "output"); err != nil {
return err
@@ -3226,6 +3245,13 @@ flow-status 取值:1=处理中(PROCESSING),2=输入中(INPUTTING),3=完成
_ = chatMessageDownloadMediaCmd.MarkFlagRequired("open-conversation-id")
chatMessageDownloadMediaCmd.Flags().String("message-id", "", "消息 openMessageId (必填)")
_ = chatMessageDownloadMediaCmd.MarkFlagRequired("message-id")
// Hidden aliases: agents routinely pass --msg-id / --open-message-id since
// the message-list output exposes the field as openMessageId/msgId. Accept
// them transparently instead of failing with "unknown flag".
chatMessageDownloadMediaCmd.Flags().String("msg-id", "", "--message-id 的别名")
_ = chatMessageDownloadMediaCmd.Flags().MarkHidden("msg-id")
chatMessageDownloadMediaCmd.Flags().String("open-message-id", "", "--message-id 的别名")
_ = chatMessageDownloadMediaCmd.Flags().MarkHidden("open-message-id")
chatMessageDownloadMediaCmd.Flags().String("output", "", "本地保存路径,文件或目录 (必填)")
_ = chatMessageDownloadMediaCmd.MarkFlagRequired("output")
@@ -222,6 +222,12 @@ func TestCrossPlatformCoverageChatWebhookReplyConversationAndDownloadEdges(t *te
tmp := t.TempDir()
base := []string{"message", "download-media", "--type=mediaId", "--resource-id=r", "--open-conversation-id=cid", "--message-id=mid"}
_ = runChatCoverageCommand(t, &productExampleCaller{dry: true}, append(base, "--output="+filepath.Join(tmp, "dry"))...)
for _, alias := range []string{"--msg-id=mid", "--open-message-id=mid"} {
aliasArgs := []string{"message", "download-media", "--type=mediaId", "--resource-id=r", "--open-conversation-id=cid", alias, "--output=" + filepath.Join(tmp, "alias-dry")}
_ = runChatCoverageCommand(t, &productExampleCaller{dry: true}, aliasArgs...)
}
missingMessageID := []string{"message", "download-media", "--type=mediaId", "--resource-id=r", "--open-conversation-id=cid", "--output=" + filepath.Join(tmp, "missing-id")}
_ = runChatCoverageCommand(t, &productExampleCaller{dry: true}, missingMessageID...)
for _, tc := range []struct {
step scriptedToolStep
out string
+33 -17
View File
@@ -17,6 +17,7 @@ import (
"fmt"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut/chatmsg"
)
// MessagesSend sends a text/markdown message as the current user
@@ -242,6 +243,11 @@ var MessagesList = shortcut.Shortcut{
// clean output projection. Both the list container and the per-item field names are
// probed defensively across candidate keys, so an empty or unexpected shape
// yields an empty list rather than a crash or fabricated data.
//
// Text goes through the shared chatmsg projection so card/auto-reply JSON is
// rendered readable and encrypted ciphertext is marked, and forwarded chat
// records ("聊天记录") expand their nested messages under "forwarded" instead of
// collapsing to a "[卡片]" summary.
func listMessagesProject(data map[string]any) []map[string]any {
raw := listMessagesResolveList(data)
out := make([]map[string]any, 0, len(raw))
@@ -250,29 +256,39 @@ func listMessagesProject(data map[string]any) []map[string]any {
if !ok {
continue
}
row := map[string]any{}
if v, ok := listMessagesFirst(m, "openMessageId", "openMsgId", "messageId", "msgId"); ok {
row["messageId"] = v
}
if v, ok := listMessagesFirst(m, "senderOpenDingTalkId", "senderUserId", "senderId", "senderStaffId"); ok {
row["senderId"] = v
}
if v, ok := listMessagesFirst(m, "msgType", "messageType", "type"); ok {
row["msgType"] = v
}
if v, ok := listMessagesFirst(m, "createTime", "sendTime", "gmtCreate", "messageTime"); ok {
row["createTime"] = v
}
if v, ok := listMessagesFirst(m, "text", "content", "plainText"); ok {
row["text"] = v
}
if len(row) > 0 {
if row := listMessageProjectOne(m); len(row) > 0 {
out = append(out, row)
}
}
return out
}
// listMessageProjectOne projects a single message into the native
// {messageId, senderId, msgType, createTime, text(, forwarded)} shape, reused
// recursively for forwarded chat records.
func listMessageProjectOne(m map[string]any) map[string]any {
row := map[string]any{}
if v, ok := listMessagesFirst(m, "openMessageId", "openMsgId", "messageId", "msgId"); ok {
row["messageId"] = v
}
if v, ok := listMessagesFirst(m, "senderOpenDingTalkId", "senderUserId", "senderId", "senderStaffId"); ok {
row["senderId"] = v
}
if v, ok := listMessagesFirst(m, "msgType", "messageType", "type"); ok {
row["msgType"] = v
}
if v, ok := listMessagesFirst(m, "createTime", "sendTime", "gmtCreate", "messageTime"); ok {
row["createTime"] = v
}
if text := chatmsg.Text(m); text != nil {
row["text"] = text
}
if forwarded := chatmsg.Forwarded(m, listMessageProjectOne); len(forwarded) > 0 {
row["forwarded"] = forwarded
}
return row
}
// listMessagesResolveList locates the list payload, tolerating a bare top-level
// array container or nesting one level deeper under a common envelope key.
func listMessagesResolveList(data map[string]any) []any {
@@ -0,0 +1,56 @@
// Copyright 2026 Alibaba Group
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chat
import (
"strings"
"testing"
)
const testCipher = "SwzNkAraDE6lUHUNlVT3mjFdbxL6dWvmt77XtjACdpJx9VFibzTbW9KtDbkzGOYP||2||1||1"
func TestListMessageProjectOne(t *testing.T) {
// full field mapping + forwarded expansion; an encrypted body is marked (no
// cross-conversation recovery), not leaked as base64.
row := listMessageProjectOne(map[string]any{
"openMessageId": "mid",
"senderOpenDingTalkId": "DXYZ",
"msgType": "text",
"createTime": "2026-07-19 13:37:03",
"content": testCipher,
"forwardMessages": []any{
map[string]any{"openMessageId": "c1", "senderOpenDingTalkId": "DA", "content": "子消息", "createTime": "t"},
},
})
if row["messageId"] != "mid" || row["senderId"] != "DXYZ" || row["msgType"] != "text" {
t.Fatalf("field mapping = %#v", row)
}
if row["createTime"] != "2026-07-19 13:37:03" {
t.Errorf("createTime = %v", row["createTime"])
}
if s, _ := row["text"].(string); !strings.Contains(s, "加密消息") || strings.Contains(s, "||2||1||") {
t.Errorf("encrypted text = %v, want marker", row["text"])
}
fwd, ok := row["forwarded"].([]map[string]any)
if !ok || len(fwd) != 1 || fwd[0]["messageId"] != "c1" || fwd[0]["text"] != "子消息" {
t.Errorf("forwarded = %#v", row["forwarded"])
}
// a bare message with no recognizable fields → empty row (no keys)
row = listMessageProjectOne(map[string]any{"unrelated": 1})
if len(row) != 0 {
t.Errorf("empty message row = %#v, want no keys", row)
}
}
+292
View File
@@ -0,0 +1,292 @@
// Copyright 2026 Alibaba Group
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package chatmsg holds the shared, read-only projection helpers for DingTalk
// message-list responses (list_individual_chat_message,
// list_conversation_message_v2, search_at_me_message, search_messages_by_keyword,
// list_topic_replies, …). Several shortcuts reshape those raw responses into a
// clean speaker/text/time list; centralising the fiddly bits here keeps them
// consistent and fixed in one place:
//
// - Sender: the display name lives under the bare "sender" key, forwarded
// entries carry the literal string "null", and some responses nest the
// speaker in a {name:…} object — all handled here.
// - Text: out-of-office auto-replies / cards arrive as raw rich-content JSON,
// and card/robot messages arrive as undecryptable ciphertext; CleanText
// renders the former to readable text and marks the latter, WITHOUT ever
// rewriting ordinary text that merely contains a JSON fragment.
// - Forwarded: a forwarded chat record ("聊天记录") hides its real per-message
// bodies in forwardMessages while the top-level content is a lossy summary.
package chatmsg
import (
"encoding/json"
"regexp"
"strings"
)
// Sender reads a message's speaker display name, tolerating common sender-name
// keys. The message-list responses carry the display name under the bare
// "sender" key (verified live), so it is probed first; the remaining aliases and
// the *Id fallbacks keep the projection resilient to other shapes. The literal
// string "null" (forwarded entries) and the empty string are treated as absent,
// and a nested {name:…} sender object yields its display name rather than the
// raw object.
func Sender(m map[string]any) any {
for _, key := range []string{"sender", "senderName", "senderNick", "nick", "senderStaffName", "userName", "name", "senderId", "senderStaffId", "senderOpenDingTalkId"} {
v, ok := m[key]
if !ok || v == nil {
continue
}
switch t := v.(type) {
case string:
if t == "" || t == "null" {
continue
}
return t
case map[string]any:
// Nested sender object: extract a display-name field; never return
// the raw map (it would surface a JSON object and block fallbacks).
if name := senderDisplayName(t); name != "" {
return name
}
continue
default:
// Scalar id (e.g. numeric) — usable as-is.
return v
}
}
return nil
}
// senderDisplayName extracts a human name from a nested sender object.
func senderDisplayName(m map[string]any) string {
for _, k := range []string{"name", "nick", "userName", "staffName", "displayName", "senderName"} {
if s, ok := m[k].(string); ok {
if s = strings.TrimSpace(s); s != "" && s != "null" {
return s
}
}
}
return ""
}
// Text reads a message's textual content (tolerating common text keys and one
// level of nesting) and runs it through CleanText.
func Text(m map[string]any) any {
for _, key := range []string{"text", "content", "msgContent", "message", "body", "plainText"} {
v, ok := m[key]
if !ok || v == nil {
continue
}
switch t := v.(type) {
case string:
if t != "" {
return CleanText(t)
}
case map[string]any:
for _, inner := range []string{"text", "content", "value"} {
if s, ok := t[inner].(string); ok && s != "" {
return CleanText(s)
}
}
}
}
return nil
}
// CreateTime reads a message's create/send time under whichever candidate key is
// present, returning the raw value.
func CreateTime(m map[string]any) any {
for _, key := range []string{"createTime", "sendTime", "gmtCreate", "createAt", "timestamp", "time"} {
if v, ok := m[key]; ok && v != nil {
return v
}
}
return nil
}
// Forwarded projects the nested messages of a forwarded chat record. The caller
// supplies its own per-message projection so each command keeps its own row
// shape; project is applied recursively, so multi-level forwards expand too.
func Forwarded(m map[string]any, project func(map[string]any) map[string]any) []map[string]any {
raw, ok := m["forwardMessages"].([]any)
if !ok || len(raw) == 0 {
return nil
}
out := make([]map[string]any, 0, len(raw))
for _, e := range raw {
if sub, ok := e.(map[string]any); ok {
out = append(out, project(sub))
}
}
return out
}
// CleanText makes a message body human-readable WITHOUT ever rewriting ordinary
// text. It only transforms a body that is a genuine DingTalk structured message:
//
// - Encrypted card/robot ciphertext (base64 + "||v||t||len" trailer) → a clear
// "[加密消息]" marker instead of the raw base64.
// - A rich-content card (out-of-office auto-reply, link/preview card, …) whose
// lines include at least one recognised rich-content block → the readable
// text extracted from those blocks, with the card's decorative JSON lines and
// "empty" placeholders dropped.
//
// Crucially, if NO line is a recognised rich-content block (e.g. ordinary text
// that merely embeds a `{"approved":false}` fragment), the original string is
// returned verbatim — a JSON line is never silently dropped.
func CleanText(s string) string {
if IsEncrypted(s) {
return "[加密消息,无法解码]"
}
// Fast path: no JSON delimiters at all — the overwhelming common case.
if !strings.ContainsAny(s, "{[") {
return s
}
lines := strings.Split(s, "\n")
isJSON := make([]bool, len(lines))
isDecoration := make([]bool, len(lines))
extracted := make([][]string, len(lines))
anyExtracted := false
for i, line := range lines {
t := strings.TrimSpace(line)
if !strings.HasPrefix(t, "{") && !strings.HasPrefix(t, "[") {
continue
}
var v any
if json.Unmarshal([]byte(t), &v) != nil {
continue
}
isJSON[i] = true
isDecoration[i] = isKnownRichDecoration(v)
if texts := richItemTexts(v); len(texts) > 0 {
extracted[i] = texts
anyExtracted = true
}
}
// No recognised rich-content block anywhere → treat the whole body as plain
// text (which may merely contain a JSON fragment) and return it untouched.
if !anyExtracted {
return s
}
out := make([]string, 0, len(lines))
for i, line := range lines {
if len(extracted[i]) > 0 {
out = append(out, extracted[i]...)
continue
}
// In card mode, drop only JSON shapes known to be card decoration.
// Unrecognised JSON may be user-authored message content and must remain
// verbatim even when another line contains a rich-content block.
if isJSON[i] && isDecoration[i] {
continue
}
if t := strings.TrimSpace(line); t == "" || t == "empty" {
continue
}
out = append(out, line)
}
// anyExtracted is true here, so out always holds at least one non-empty
// extracted text — the joined result is never empty.
return strings.TrimSpace(strings.Join(out, "\n"))
}
// isKnownRichDecoration recognises the two decoration records emitted alongside
// DingTalk rich-content bodies. Keep this deliberately narrow: an arbitrary JSON
// object in the same message is user content unless its shape is known here.
func isKnownRichDecoration(node any) bool {
m, ok := node.(map[string]any)
if !ok {
return false
}
_, hasPreviewURL := m["previewUrl"]
_, hasTitle := m["title"]
_, hasAutoLayout := m["autoLayout"]
_, hasEnableForward := m["enableForward"]
return (hasPreviewURL && hasTitle) || (hasAutoLayout && hasEnableForward)
}
// richItemTexts walks a decoded DingTalk rich-content blob and returns the
// readable text carried by its rich-content items (items[].data.text). It only
// harvests item bodies, so decorative fields (card titles, preview URLs, layout
// config) contribute nothing and are dropped. An empty result means "not a
// recognised rich-content block".
func richItemTexts(node any) []string {
var texts []string
var walk func(n any)
walk = func(n any) {
switch t := n.(type) {
case []any:
for _, e := range t {
walk(e)
}
case map[string]any:
if items, ok := t["items"].([]any); ok {
for _, it := range items {
mm, ok := it.(map[string]any)
if !ok {
continue
}
data, ok := mm["data"].(map[string]any)
if !ok {
continue
}
if s, ok := data["text"].(string); ok {
if s = strings.TrimSpace(s); s != "" {
texts = append(texts, s)
}
}
}
}
for _, e := range t {
walk(e)
}
}
}
walk(node)
return texts
}
// encryptedTrailerRE matches DingTalk's encrypted-message trailer
// "||<version>||<type>||<len>" (e.g. "||2||1||196") anchored at the end.
var encryptedTrailerRE = regexp.MustCompile(`\|\|\d+\|\|\d+\|\|\d+\s*$`)
// IsEncrypted reports whether a message body is a raw DingTalk encrypted-message
// ciphertext: a base64 blob (DingTalk wraps it across several lines) followed by
// the "||v||t||len" trailer. It is intentionally strict — both the trailer and a
// pure-base64 body are required — so ordinary text (CJK, punctuation, …) never
// trips it.
func IsEncrypted(s string) bool {
s = strings.TrimSpace(s)
if !encryptedTrailerRE.MatchString(s) {
return false
}
body := strings.TrimSpace(encryptedTrailerRE.ReplaceAllString(s, ""))
if len(body) < 32 {
return false
}
for _, r := range body {
switch {
case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9',
r == '+', r == '/', r == '=', r == '\n', r == '\r', r == ' ', r == '\t':
default:
return false
}
}
return true
}
+176
View File
@@ -0,0 +1,176 @@
// Copyright 2026 Alibaba Group
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chatmsg
import (
"strings"
"testing"
)
func TestSender(t *testing.T) {
// The display name lives under the bare "sender" key.
if got := Sender(map[string]any{"sender": "念晨", "senderOpenDingTalkId": "D1"}); got != "念晨" {
t.Fatalf("sender = %v, want 念晨", got)
}
// Falls back to the open id when no display name is present.
if got := Sender(map[string]any{"senderOpenDingTalkId": "DXYZ"}); got != "DXYZ" {
t.Fatalf("sender fallback = %v, want DXYZ", got)
}
// forwardMessages entries carry the literal string "null" — treat as absent.
if got := Sender(map[string]any{"sender": "null"}); got != nil {
t.Fatalf("sender \"null\" = %v, want nil", got)
}
if got := Sender(map[string]any{"sender": "null", "senderName": "念晨"}); got != "念晨" {
t.Fatalf("sender \"null\" fallthrough = %v, want 念晨", got)
}
// A nested {name:…} sender object yields its display name, not the raw map.
if got := Sender(map[string]any{"sender": map[string]any{"name": "Alice"}}); got != "Alice" {
t.Fatalf("nested sender = %v, want Alice", got)
}
// A nested sender object with no usable name must not block the fallback.
if got := Sender(map[string]any{"sender": map[string]any{"foo": "bar"}, "senderName": "Bob"}); got != "Bob" {
t.Fatalf("nested-no-name fallthrough = %v, want Bob", got)
}
// A scalar numeric id is returned as-is.
if got := Sender(map[string]any{"senderId": float64(42)}); got != float64(42) {
t.Fatalf("numeric sender id = %v", got)
}
}
func TestCleanText(t *testing.T) {
// Out-of-office auto-reply: readable body lives in items[].data.text; the
// decorative preview/config JSON lines and "empty" placeholder are dropped.
autoReply := "* 仅你和对方可见\n" +
`[{"text":{"minSupportVersion":"1.1","translateMap":{},"version":"1.2","items":[{"fallbackKey":"","data":{"text":"你好,我在出差中,消息回复可能不及时。"},"style":{"size":15,"bold":0},"type":"text"}]},"type":"markdown"}]` + "\n" +
`{"previewUrl":"dingtalk://x","title":{"text":"自动回复","type":"text"}}` + "\n" +
"empty\n" +
`{"autoLayout":false,"enableForward":false}`
if got, want := CleanText(autoReply), "* 仅你和对方可见\n你好,我在出差中,消息回复可能不及时。"; got != want {
t.Fatalf("auto-reply cleaned = %q, want %q", got, want)
}
// P1 regression: ordinary text whose middle line is a JSON fragment (no
// rich-content block anywhere) must be returned VERBATIM, not rewritten.
mixed := "payload:\n{\"approved\":false}\nplease check"
if got := CleanText(mixed); got != mixed {
t.Fatalf("mixed text was rewritten: got %q, want %q", got, mixed)
}
// An ordinary JSON line must also survive when a different line contains a
// recognised rich-content block. Card mode is not permission to discard
// unrelated user-authored JSON.
richAndPlain := `[{"items":[{"data":{"text":"卡片正文"}}]}]` + "\n" +
`{"approved":false}`
if got, want := CleanText(richAndPlain), "卡片正文\n{\"approved\":false}"; got != want {
t.Fatalf("mixed rich/plain JSON was rewritten: got %q, want %q", got, want)
}
// Malformed items (non-map item, item whose "data" isn't a map) are skipped;
// only the well-formed item's text is extracted.
blob := `[{"items":["notmap",{"data":"notmap"},{"data":{"text":"有效正文"}}]}]`
if got := CleanText(blob); got != "有效正文" {
t.Fatalf("CleanText rich edge = %q, want 有效正文", got)
}
tests := map[string]string{
"上周五 7.1 KW": "上周五 7.1 KW",
"上周客户统计的[图片消息](mediaId=@lQ)": "上周客户统计的[图片消息](mediaId=@lQ)",
"[文件] 简历.pdf fileId: qnY 注意:如需下载使用dws drive download命令下载": "[文件] 简历.pdf fileId: qnY 注意:如需下载使用dws drive download命令下载",
"[讨论] 排期\n明天开会": "[讨论] 排期\n明天开会",
// a lone JSON object that isn't a rich-content block is left untouched
`{"autoLayout":false,"enableForward":false}`: `{"autoLayout":false,"enableForward":false}`,
}
for in, want := range tests {
if got := CleanText(in); got != want {
t.Errorf("CleanText(%q) = %q, want %q", in, got, want)
}
}
}
func TestIsEncryptedAndMarker(t *testing.T) {
cipher := "SwzNkAraDE6lUHUNlVT3mjFdbxL6dWvmt77XtjACdpJx9VFibzTbW9KtDbkzGOYP\n" +
"7oDptklFO+YzDltH+myErV6rkc8URHYykpeSDsMP6kznFa9E320NsIntfY771dx+\n" +
"||2||1||196"
if !IsEncrypted(cipher) {
t.Fatalf("ciphertext not detected: %q", cipher)
}
if got := CleanText(cipher); !strings.Contains(got, "加密消息") || strings.Contains(got, "||2||1||") {
t.Fatalf("encrypted cleaned = %q, want marker not ciphertext", got)
}
for _, s := range []string{
"上周五 7.1 KW",
"价格 100||2||1||3",
"[图片消息](mediaId=@lQLPJwDw3VmNDcfMos0DhLB3OHPQeTBlzgov2Oi1ly4A)",
"大哥,我看了一下我觉得有几个点可以关注一下",
strings.Repeat("好", 20) + "||2||1||1", // long CJK body + trailer, not base64
} {
if IsEncrypted(s) {
t.Errorf("false positive: %q flagged as encrypted", s)
}
}
}
func TestText(t *testing.T) {
if got := Text(map[string]any{"content": "你好"}); got != "你好" {
t.Errorf("Text string = %v", got)
}
if got := Text(map[string]any{"content": map[string]any{"text": "嵌套"}}); got != "嵌套" {
t.Errorf("Text nested = %v", got)
}
if got := Text(map[string]any{"plainText": "纯文本"}); got != "纯文本" {
t.Errorf("Text plainText = %v", got)
}
if got := Text(map[string]any{"foo": 1}); got != nil {
t.Errorf("Text none = %v, want nil", got)
}
}
func TestCreateTime(t *testing.T) {
if got := CreateTime(map[string]any{"sendTime": "2026-07-19 13:37:03"}); got != "2026-07-19 13:37:03" {
t.Errorf("CreateTime = %v", got)
}
if got := CreateTime(map[string]any{}); got != nil {
t.Errorf("CreateTime empty = %v, want nil", got)
}
}
func TestForwarded(t *testing.T) {
var project func(m map[string]any) map[string]any
project = func(m map[string]any) map[string]any {
row := map[string]any{"text": Text(m)}
if fwd := Forwarded(m, project); len(fwd) > 0 { // recurse
row["forwarded"] = fwd
}
return row
}
if got := Forwarded(map[string]any{"content": "x"}, project); got != nil {
t.Errorf("Forwarded none = %v", got)
}
fwd := Forwarded(map[string]any{
"forwardMessages": []any{
map[string]any{"content": "a"},
"not-a-map",
map[string]any{"content": "b", "forwardMessages": []any{
map[string]any{"content": "nested"},
}},
},
}, project)
if len(fwd) != 2 || fwd[0]["text"] != "a" || fwd[1]["text"] != "b" {
t.Fatalf("Forwarded = %#v", fwd)
}
nested, ok := fwd[1]["forwarded"].([]map[string]any)
if !ok || len(nested) != 1 || nested[0]["text"] != "nested" {
t.Errorf("nested forwarded = %#v", fwd[1]["forwarded"])
}
}
+41 -11
View File
@@ -19,6 +19,7 @@ import (
"time"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut/chatmsg"
)
// AtMe: pull the messages that recently @-mentioned ME across chats in one step.
@@ -91,12 +92,7 @@ var AtMe = shortcut.Shortcut{
}
results := make([]map[string]any, 0, len(items))
for _, m := range items {
results = append(results, map[string]any{
"sender": atMeSender(m),
"time": atMeTime(m),
"text": atMeText(m),
"conversation": atMeConversation(m),
})
results = append(results, atMeProject(m))
}
return rt.Output(map[string]any{"messages": results})
},
@@ -194,28 +190,62 @@ func atMeToMaps(arr []any) []map[string]any {
return out
}
// atMeProject reshapes one @me message into {sender, time, text, conversation},
// running text through the shared chatmsg cleaning (card/auto-reply JSON →
// readable, ciphertext → marker) and recursively expanding any forwarded chat
// record under "forwarded".
func atMeProject(m map[string]any) map[string]any {
row := map[string]any{
"sender": atMeSender(m),
"time": atMeTime(m),
"text": atMeCleanText(m),
"conversation": atMeConversation(m),
}
if forwarded := chatmsg.Forwarded(m, atMeProject); len(forwarded) > 0 {
row["forwarded"] = forwarded
}
return row
}
// atMeCleanText runs atMeText's extraction through chatmsg.CleanText so
// card/auto-reply JSON and ciphertext render readable instead of leaking raw.
func atMeCleanText(m map[string]any) any {
if s, ok := atMeText(m).(string); ok {
return chatmsg.CleanText(s)
}
return atMeText(m)
}
// atMeSender reads a message's sender display name/id, tolerating the common
// sender keys the gateway may use (including a nested sender object).
// sender keys the gateway may use (including a nested sender object). The literal
// string "null" (carried by forwarded sub-messages) and the empty string are
// both treated as absent so they never surface as the speaker.
func atMeSender(m map[string]any) any {
norm := func(v any) string {
if s := atMeString(v); s != "" && s != "null" {
return s
}
return ""
}
for _, key := range []string{"senderName", "sender_name", "senderNick", "fromName", "senderStaffName"} {
if v := atMeString(m[key]); v != "" {
if v := norm(m[key]); v != "" {
return v
}
}
for _, key := range []string{"sender", "from", "senderUser"} {
if nested, ok := m[key].(map[string]any); ok {
for _, k2 := range []string{"name", "nick", "userName", "staffName", "displayName"} {
if v := atMeString(nested[k2]); v != "" {
if v := norm(nested[k2]); v != "" {
return v
}
}
}
if v := atMeString(m[key]); v != "" {
if v := norm(m[key]); v != "" {
return v
}
}
for _, key := range []string{"senderId", "sender_id", "senderUserId", "senderStaffId", "openDingTalkId"} {
if v := atMeString(m[key]); v != "" {
if v := norm(m[key]); v != "" {
return v
}
}
@@ -0,0 +1,131 @@
// Copyright 2026 Alibaba Group
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package smart
import (
"strings"
"testing"
)
const testCipher = "SwzNkAraDE6lUHUNlVT3mjFdbxL6dWvmt77XtjACdpJx9VFibzTbW9KtDbkzGOYP||2||1||1"
func TestAtMeProject(t *testing.T) {
// nested sender object + plain text
row := atMeProject(map[string]any{
"sender": map[string]any{"name": "念晨"},
"createTime": "2026-07-19 13:37:03",
"content": "普通消息",
"conversationTitle": "群A",
"openConversationId": "cid1",
})
if row["sender"] != "念晨" || row["text"] != "普通消息" || row["conversation"] != "群A" {
t.Fatalf("atMeProject nested = %#v", row)
}
// encrypted content → marked (never leaked); id-only sender fallback; a
// forwarded sub-message whose sender is the literal "null" must be nulled.
row = atMeProject(map[string]any{
"senderId": "DXYZ",
"openMessageId": "m1",
"content": testCipher,
"forwardMessages": []any{
map[string]any{"sender": "null", "content": "子消息", "createTime": "t"},
},
})
if row["sender"] != "DXYZ" {
t.Errorf("atMeProject id-fallback sender = %v", row["sender"])
}
if s, _ := row["text"].(string); !strings.Contains(s, "加密消息") {
t.Errorf("atMeProject encrypted text = %v, want marker", row["text"])
}
fwd, ok := row["forwarded"].([]map[string]any)
if !ok || len(fwd) != 1 {
t.Fatalf("atMeProject forwarded = %#v", row["forwarded"])
}
if fwd[0]["sender"] != nil {
t.Errorf("forwarded sub sender = %v, want nil (literal \"null\")", fwd[0]["sender"])
}
// no sender / no text at all → nils, no forwarded key
row = atMeProject(map[string]any{"createTime": "t"})
if row["sender"] != nil || row["text"] != nil {
t.Errorf("atMeProject empty = %#v", row)
}
if _, has := row["forwarded"]; has {
t.Errorf("atMeProject plain unexpectedly has forwarded")
}
}
func TestSearchMsgProject(t *testing.T) {
// nested sender + plain text + messageId
row := searchMsgProject(map[string]any{
"sender": map[string]any{"nick": "千启"},
"createTime": "2026-07-19 13:37:03",
"content": "命中关键词的消息",
"msgId": "mid1",
})
if row["sender"] != "千启" || row["text"] != "命中关键词的消息" {
t.Fatalf("searchMsgProject = %#v", row)
}
// encrypted → marker; id-only sender; forwarded "null" sender nulled.
row = searchMsgProject(map[string]any{
"senderId": "DAAA",
"openMessageId": "m2",
"content": testCipher,
"forwardMessages": []any{
map[string]any{"sender": "null", "content": "转发子消息", "createTime": "t"},
},
})
if row["sender"] != "DAAA" {
t.Errorf("searchMsgProject sender = %v", row["sender"])
}
if s, _ := row["text"].(string); !strings.Contains(s, "加密消息") {
t.Errorf("searchMsgProject encrypted text = %v, want marker", row["text"])
}
fwd, ok := row["forwarded"].([]map[string]any)
if !ok || len(fwd) != 1 || fwd[0]["sender"] != nil {
t.Errorf("searchMsgProject forwarded = %#v", row["forwarded"])
}
// no sender / no text
row = searchMsgProject(map[string]any{"createTime": "t"})
if row["sender"] != nil || row["text"] != nil {
t.Errorf("searchMsgProject empty = %#v", row)
}
}
// TestSenderHelpers exercises the atMe/searchMsg sender key families directly:
// a senderName-family key (first probe loop), a flat string under "sender"
// (second loop), and the "null" sentinel normalisation.
func TestSenderHelpers(t *testing.T) {
cases := []struct {
fn func(map[string]any) any
name string
}{
{atMeSender, "atMeSender"},
{searchMsgSender, "searchMsgSender"},
}
for _, c := range cases {
if got := c.fn(map[string]any{"senderName": "张三"}); got != "张三" {
t.Errorf("%s senderName = %v, want 张三", c.name, got)
}
if got := c.fn(map[string]any{"sender": "李四"}); got != "李四" {
t.Errorf("%s flat sender = %v, want 李四", c.name, got)
}
if got := c.fn(map[string]any{"senderName": "null"}); got != nil {
t.Errorf("%s \"null\" = %v, want nil", c.name, got)
}
}
}
+14 -50
View File
@@ -17,6 +17,7 @@ import (
"strings"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut/chatmsg"
)
// ChatMessages: fetch the message list of one conversation (group OR single
@@ -111,11 +112,7 @@ var ChatMessages = shortcut.Shortcut{
items := chatMessageItems(data)
results := make([]map[string]any, 0, len(items))
for _, m := range items {
results = append(results, map[string]any{
"sender": chatMessageSender(m),
"text": chatMessageText(m),
"createTime": chatMessageCreateTime(m),
})
results = append(results, projectChatMessage(m))
}
return rt.Output(map[string]any{
@@ -156,53 +153,20 @@ func chatMessageItems(data map[string]any) []map[string]any {
return nil
}
// chatMessageSender reads a message's speaker display name, tolerating common
// sender-name keys.
func chatMessageSender(m map[string]any) any {
for _, key := range []string{"senderName", "senderNick", "nick", "senderStaffName", "userName", "name", "senderId", "senderStaffId"} {
if v, ok := m[key]; ok && v != nil {
if s, ok := v.(string); ok && s == "" {
continue
}
return v
}
// projectChatMessage reshapes one raw message into the clean
// {sender, text, createTime} projection, rendering card/auto-reply JSON and
// marking encrypted messages via chatmsg, and recursively expanding forwarded
// chat records under "forwarded".
func projectChatMessage(m map[string]any) map[string]any {
row := map[string]any{
"sender": chatmsg.Sender(m),
"text": chatmsg.Text(m),
"createTime": chatmsg.CreateTime(m),
}
return nil
}
// chatMessageText reads a message's textual content, tolerating common text
// keys and one level of nesting (e.g. {"content":{"text":"..."}}).
func chatMessageText(m map[string]any) any {
for _, key := range []string{"text", "content", "msgContent", "message", "body"} {
v, ok := m[key]
if !ok || v == nil {
continue
}
switch t := v.(type) {
case string:
if t != "" {
return t
}
case map[string]any:
for _, inner := range []string{"text", "content", "value"} {
if s, ok := t[inner].(string); ok && s != "" {
return s
}
}
}
if forwarded := chatmsg.Forwarded(m, projectChatMessage); len(forwarded) > 0 {
row["forwarded"] = forwarded
}
return nil
}
// chatMessageCreateTime reads a message's create/send time, returning the raw
// value under whichever candidate key is present.
func chatMessageCreateTime(m map[string]any) any {
for _, key := range []string{"createTime", "sendTime", "gmtCreate", "createAt", "timestamp", "time"} {
if v, ok := m[key]; ok && v != nil {
return v
}
}
return nil
return row
}
func init() {
@@ -0,0 +1,62 @@
// Copyright 2026 Alibaba Group
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package smart
import "testing"
// TestProjectChatMessageExpandsForwarded guards that a forwarded chat record
// ("聊天记录") exposes its nested messages under "forwarded" instead of
// collapsing to the lossy top-level "[卡片]" summary, recursing through nested
// forwards, and that the string-"null" sender is nulled out. The per-field
// behaviour (sender/text/encryption) is covered in the chatmsg package tests.
func TestProjectChatMessageExpandsForwarded(t *testing.T) {
row := projectChatMessage(map[string]any{
"sender": "hugozhu",
"content": "hugozhu与opencode-agent的聊天记录\nopencode-agent:[卡片]",
"createTime": "2026-07-20 21:41:21",
"forwardMessages": []any{
map[string]any{"sender": "null", "content": "读下冬翔发给我的最近两条消息", "createTime": "2026-07-20 09:30:33"},
map[string]any{"sender": "冬翔", "content": "W29 工作总结", "createTime": "2026-07-19 23:35:40",
// nested forward inside a forward — must expand recursively.
"forwardMessages": []any{
map[string]any{"sender": "念晨", "content": "收到", "createTime": "2026-07-19 23:36:00"},
},
},
},
})
if row["sender"] != "hugozhu" {
t.Fatalf("top sender = %v, want hugozhu", row["sender"])
}
forwarded, ok := row["forwarded"].([]map[string]any)
if !ok || len(forwarded) != 2 {
t.Fatalf("forwarded = %#v, want 2 entries", row["forwarded"])
}
if forwarded[0]["sender"] != nil {
t.Errorf("forwarded[0].sender = %v, want nil (string \"null\")", forwarded[0]["sender"])
}
if forwarded[0]["text"] != "读下冬翔发给我的最近两条消息" {
t.Errorf("forwarded[0].text = %v", forwarded[0]["text"])
}
nested, ok := forwarded[1]["forwarded"].([]map[string]any)
if !ok || len(nested) != 1 || nested[0]["sender"] != "念晨" {
t.Errorf("nested forwarded = %#v, want 1 entry from 念晨", forwarded[1]["forwarded"])
}
// A plain message must not grow a "forwarded" key.
plain := projectChatMessage(map[string]any{"sender": "念晨", "content": "hi", "createTime": "t"})
if _, has := plain["forwarded"]; has {
t.Errorf("plain message unexpectedly has forwarded key: %#v", plain)
}
}
+40 -11
View File
@@ -19,6 +19,7 @@ import (
"time"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut"
"github.com/DingTalk-Real-AI/dingtalk-workspace-cli/internal/shortcut/chatmsg"
)
// SearchMsg: search messages inside a single group chat by keyword in one step.
@@ -108,12 +109,7 @@ var SearchMsg = shortcut.Shortcut{
}
results := make([]map[string]any, 0, len(items))
for _, m := range items {
results = append(results, map[string]any{
"sender": searchMsgSender(m),
"time": searchMsgTime(m),
"text": searchMsgText(m),
"messageId": searchMsgMessageID(m),
})
results = append(results, searchMsgProject(m))
}
return rt.Output(map[string]any{"messages": results})
},
@@ -152,28 +148,61 @@ func searchMsgToMaps(arr []any) []map[string]any {
return out
}
// searchMsgProject reshapes one matched message into {sender, time, text,
// messageId}, running text through the shared chatmsg cleaning (card/auto-reply
// JSON → readable, ciphertext → marker) and recursively expanding any forwarded
// chat record under "forwarded".
func searchMsgProject(m map[string]any) map[string]any {
row := map[string]any{
"sender": searchMsgSender(m),
"time": searchMsgTime(m),
"text": searchMsgCleanText(m),
"messageId": searchMsgMessageID(m),
}
if forwarded := chatmsg.Forwarded(m, searchMsgProject); len(forwarded) > 0 {
row["forwarded"] = forwarded
}
return row
}
// searchMsgCleanText runs searchMsgText's extraction through chatmsg.CleanText.
func searchMsgCleanText(m map[string]any) any {
if s, ok := searchMsgText(m).(string); ok {
return chatmsg.CleanText(s)
}
return searchMsgText(m)
}
// searchMsgSender reads a message's sender display name/id, tolerating the
// common sender keys the gateway may use (including a nested sender object).
// common sender keys the gateway may use (including a nested sender object). The
// literal string "null" (carried by forwarded sub-messages) and the empty string
// are both treated as absent so they never surface as the speaker.
func searchMsgSender(m map[string]any) any {
norm := func(v any) string {
if s := searchMsgString(v); s != "" && s != "null" {
return s
}
return ""
}
for _, key := range []string{"senderName", "sender_name", "senderNick", "fromName", "senderStaffName"} {
if v := searchMsgString(m[key]); v != "" {
if v := norm(m[key]); v != "" {
return v
}
}
for _, key := range []string{"sender", "from", "senderUser"} {
if nested, ok := m[key].(map[string]any); ok {
for _, k2 := range []string{"name", "nick", "userName", "staffName", "displayName"} {
if v := searchMsgString(nested[k2]); v != "" {
if v := norm(nested[k2]); v != "" {
return v
}
}
}
if v := searchMsgString(m[key]); v != "" {
if v := norm(m[key]); v != "" {
return v
}
}
for _, key := range []string{"senderId", "sender_id", "senderUserId", "senderStaffId", "openDingTalkId"} {
if v := searchMsgString(m[key]); v != "" {
if v := norm(m[key]); v != "" {
return v
}
}
+1 -54
View File
@@ -78,11 +78,7 @@ var ThreadReplies = shortcut.Shortcut{
items := threadReplyItems(data)
results := make([]map[string]any, 0, len(items))
for _, m := range items {
results = append(results, map[string]any{
"sender": threadReplySender(m),
"text": threadReplyText(m),
"createTime": threadReplyCreateTime(m),
})
results = append(results, projectChatMessage(m))
}
return rt.Output(map[string]any{
@@ -124,55 +120,6 @@ func threadReplyItems(data map[string]any) []map[string]any {
return nil
}
// threadReplySender reads a reply's speaker display name, tolerating common
// sender-name keys.
func threadReplySender(m map[string]any) any {
for _, key := range []string{"senderName", "senderNick", "nick", "senderStaffName", "userName", "name", "senderId", "senderStaffId"} {
if v, ok := m[key]; ok && v != nil {
if s, ok := v.(string); ok && s == "" {
continue
}
return v
}
}
return nil
}
// threadReplyText reads a reply's textual content, tolerating common text keys
// and one level of nesting (e.g. {"text":{"content":"..."}}).
func threadReplyText(m map[string]any) any {
for _, key := range []string{"text", "content", "msgContent", "message", "body"} {
v, ok := m[key]
if !ok || v == nil {
continue
}
switch t := v.(type) {
case string:
if t != "" {
return t
}
case map[string]any:
for _, inner := range []string{"content", "text", "value"} {
if s, ok := t[inner].(string); ok && s != "" {
return s
}
}
}
}
return nil
}
// threadReplyCreateTime reads a reply's create/send time, returning the raw
// value under whichever candidate key is present.
func threadReplyCreateTime(m map[string]any) any {
for _, key := range []string{"createTime", "sendTime", "gmtCreate", "createAt", "timestamp", "time"} {
if v, ok := m[key]; ok && v != nil {
return v
}
}
return nil
}
func init() {
shortcut.Register(ThreadReplies)
}
+116 -13
View File
@@ -8,16 +8,32 @@ GITEE_API="${GITEE_API:-https://gitee.com/api/v5}"
GITEE_CURL_CONNECT_TIMEOUT="${GITEE_CURL_CONNECT_TIMEOUT:-15}"
GITEE_CURL_MAX_TIME="${GITEE_CURL_MAX_TIME:-120}"
GITEE_LIST_MAX_TIME="${GITEE_LIST_MAX_TIME:-20}"
# Attachment-list reads are safe to retry and are used both before an upload
# and to detect a committed upload whose HTTP response was lost. Keep retrying
# long enough to bridge a short Gitee TLS/API outage without ever blindly
# replaying the upload itself.
GITEE_LIST_RETRIES="${GITEE_LIST_RETRIES:-24}"
GITEE_LIST_RETRY_DELAY="${GITEE_LIST_RETRY_DELAY:-20}"
GITEE_LIST_RETRY_WINDOW_SECONDS="${GITEE_LIST_RETRY_WINDOW_SECONDS:-420}"
GITEE_VERIFY_MAX_TIME="${GITEE_VERIFY_MAX_TIME:-60}"
GITEE_MUTATION_MAX_TIME="${GITEE_MUTATION_MAX_TIME:-20}"
GITEE_UPLOAD_MAX_TIME="${GITEE_UPLOAD_MAX_TIME:-300}"
# Gitee does not return a response until an attachment upload is committed.
# From GitHub-hosted runners, a near-10 MiB DWS binary can legitimately take
# more than five minutes, so keep the transfer deadline above that observed
# floor while the per-asset and overall deadlines retain hard upper bounds.
GITEE_UPLOAD_MAX_TIME="${GITEE_UPLOAD_MAX_TIME:-1200}"
# The per-asset deadline guarantees one complete slow upload plus one complete
# attachment-list recovery on each side of that upload. The second upload
# attempt is allowed only for zero-byte failures before HTTP begins; a first
# attempt that consumes the full transfer deadline intentionally exhausts the
# retry budget instead of holding a runner for another full slow upload.
GITEE_UPLOAD_RETRIES="${GITEE_UPLOAD_RETRIES:-2}"
GITEE_UPLOAD_RETRY_DELAY="${GITEE_UPLOAD_RETRY_DELAY:-5}"
GITEE_EXISTING_VERIFY_ATTEMPTS="${GITEE_EXISTING_VERIFY_ATTEMPTS:-1}"
GITEE_POST_UPLOAD_VERIFY_ATTEMPTS="${GITEE_POST_UPLOAD_VERIFY_ATTEMPTS:-2}"
GITEE_VERIFY_RETRY_DELAY="${GITEE_VERIFY_RETRY_DELAY:-5}"
GITEE_ASSET_TIMEOUT_SECONDS="${GITEE_ASSET_TIMEOUT_SECONDS:-600}"
GITEE_OVERALL_TIMEOUT_SECONDS="${GITEE_OVERALL_TIMEOUT_SECONDS:-4920}"
GITEE_ASSET_TIMEOUT_SECONDS="${GITEE_ASSET_TIMEOUT_SECONDS:-2220}"
GITEE_OVERALL_TIMEOUT_SECONDS="${GITEE_OVERALL_TIMEOUT_SECONDS:-18300}"
err() {
printf 'error: %s\n' "$*" >&2
@@ -43,6 +59,8 @@ for setting in \
GITEE_CURL_CONNECT_TIMEOUT \
GITEE_CURL_MAX_TIME \
GITEE_LIST_MAX_TIME \
GITEE_LIST_RETRIES \
GITEE_LIST_RETRY_WINDOW_SECONDS \
GITEE_VERIFY_MAX_TIME \
GITEE_MUTATION_MAX_TIME \
GITEE_UPLOAD_MAX_TIME \
@@ -54,6 +72,7 @@ for setting in \
require_positive_integer "$setting" "${!setting}"
done
for setting in \
GITEE_LIST_RETRY_DELAY \
GITEE_UPLOAD_RETRY_DELAY \
GITEE_VERIFY_RETRY_DELAY; do
require_nonnegative_integer "$setting" "${!setting}"
@@ -139,21 +158,81 @@ api_get() {
--max-time "$max_time" "$@"
}
list_assets() {
api_get "$GITEE_LIST_MAX_TIME" \
list_assets_once() {
local max_time="$1" payload
payload="$(api_get "$max_time" \
-H "Authorization: token ${GITEE_TOKEN}" \
"${base}/releases/${release_id}/attach_files" \
| python3 -c 'import json,sys
"${base}/releases/${release_id}/attach_files")" || return 1
printf '%s\n' "$payload" | python3 -c 'import json,sys
data=json.load(sys.stdin)
rows=data if isinstance(data,list) else data.get("attach_files",[])
if isinstance(data,list):
rows=data
elif isinstance(data,dict) and "attach_files" in data:
rows=data["attach_files"]
else:
raise ValueError("attachment list response must be a list or contain attach_files")
if not isinstance(rows,list):
raise ValueError("attach_files must be a list")
for asset in rows:
if not isinstance(asset,dict):
raise ValueError("each attachment must be an object")
name=asset.get("name","")
asset_id=asset.get("id","")
url=asset.get("browser_download_url","")
if name and asset_id != "":
print("%s\t%s\t%s" % (name, asset_id, url))'
valid_id=(isinstance(asset_id,int) and not isinstance(asset_id,bool) and asset_id > 0) or (isinstance(asset_id,str) and asset_id.isdigit() and int(asset_id) > 0)
if not isinstance(name,str) or not name or not valid_id or not isinstance(url,str) or not url:
raise ValueError("attachment identity and download URL are required")
print("%s\t%s\t%s" % (name, asset_id, url))'
}
list_assets() {
local attempt=1 candidate now retry_deadline remaining max_time delay
now="$(now_seconds)"
retry_deadline=$(( now + GITEE_LIST_RETRY_WINDOW_SECONDS ))
if [ "$retry_deadline" -gt "$active_deadline" ]; then
retry_deadline="$active_deadline"
fi
while [ "$attempt" -le "$GITEE_LIST_RETRIES" ]; do
now="$(now_seconds)"
remaining=$(( retry_deadline - now ))
[ "$remaining" -gt 0 ] || break
max_time="$GITEE_LIST_MAX_TIME"
if [ "$max_time" -gt "$remaining" ]; then
max_time="$remaining"
fi
# Publish one complete parse atomically. A malformed response can make the
# parser emit valid leading rows before it fails; leaking those rows into a
# later successful retry would manufacture duplicates and could delete a
# correct attachment.
if candidate="$(list_assets_once "$max_time")"; then
now="$(now_seconds)"
if [ "$now" -lt "$retry_deadline" ]; then
printf '%s\n' "$candidate"
return 0
fi
break
fi
if [ "$attempt" -ge "$GITEE_LIST_RETRIES" ]; then
break
fi
attempt=$((attempt + 1))
now="$(now_seconds)"
remaining=$(( retry_deadline - now ))
[ "$remaining" -gt 0 ] || break
delay="$GITEE_LIST_RETRY_DELAY"
if [ "$delay" -gt 0 ]; then
# Do not turn a configured backoff into a burst of zero-delay requests
# during the final wall-clock second. Explicit delay=0 remains available
# to fast unit tests and tightly controlled callers.
[ "$remaining" -gt 1 ] || break
if [ "$delay" -ge "$remaining" ]; then
delay=$(( remaining - 1 ))
fi
fi
echo " ⚠ Gitee attachment list attempt $((attempt - 1))/${GITEE_LIST_RETRIES} failed; retrying in ${delay}s" >&2
sleep_within_deadline "$delay" || return 1
done
return 1
}
sha256_of() {
@@ -234,22 +313,42 @@ delete_named_assets() {
}
gitee_attach() {
local file="$1" name attempt response status max_time
local file="$1" name attempt response status max_time transfer_log metrics http_code size_upload safe_to_retry
name="$(basename "$file")"
attempt=1
while [ "$attempt" -le "$GITEE_UPLOAD_RETRIES" ]; do
status=0
max_time="$(bounded_max_time "$GITEE_UPLOAD_MAX_TIME")" || return 1
if response="$(curl -fsS --connect-timeout "$GITEE_CURL_CONNECT_TIMEOUT" \
transfer_log="$(mktemp)" || return 1
if metrics="$(curl -fsS --connect-timeout "$GITEE_CURL_CONNECT_TIMEOUT" \
--max-time "$max_time" \
-X POST "${base}/releases/${release_id}/attach_files" \
-H "Authorization: token ${GITEE_TOKEN}" \
-H "Expect:" \
-F "file=@${file}" 2>&1)"; then
-F "file=@${file}" \
-o /dev/null \
-w '%{http_code}\t%{size_upload}' \
2>"$transfer_log")"; then
status=0
else
status=$?
fi
response="$(<"$transfer_log")"
rm -f "$transfer_log"
http_code="${metrics%%$'\t'*}"
if [ "$metrics" = "$http_code" ]; then
size_upload=""
else
size_upload="${metrics#*$'\t'}"
fi
safe_to_retry=0
if [ "$status" -ne 0 ] && [ "$http_code" = "000" ] && \
awk -v value="$size_upload" 'BEGIN { exit !((value + 0) == 0 && value ~ /^[0-9]+([.][0-9]+)?$/) }'; then
# A request that never reached HTTP and uploaded zero bytes cannot have
# committed an attachment. TLS/DNS/connect failures are therefore the
# only transport failures safe to replay automatically.
safe_to_retry=1
fi
# Gitee can commit an upload but let the HTTP response time out. Probe the
# release before retrying so a lost response does not create duplicates.
@@ -261,6 +360,10 @@ gitee_attach() {
fi
echo " ⚠ upload attempt ${attempt}/${GITEE_UPLOAD_RETRIES} failed for ${name}: $(printf '%s' "$response" | head -c 240)" >&2
if [ "$safe_to_retry" -ne 1 ]; then
echo " ⚠ ${name} upload outcome is ambiguous (HTTP ${http_code:-unknown}, uploaded ${size_upload:-unknown} bytes); refusing to replay POST" >&2
return 1
fi
attempt=$((attempt + 1))
if [ "$attempt" -le "$GITEE_UPLOAD_RETRIES" ]; then
# Remove any partial, stale, or duplicate attachment before the one
+2 -2
View File
@@ -31,13 +31,13 @@ DIST_DIR="${DIST_DIR:-dist}"
GITEE_API="${GITEE_API:-https://gitee.com/api/v5}"
GITEE_CURL_CONNECT_TIMEOUT="${GITEE_CURL_CONNECT_TIMEOUT:-15}"
GITEE_CURL_MAX_TIME="${GITEE_CURL_MAX_TIME:-120}"
GITEE_SYNC_TIMEOUT_SECONDS="${GITEE_SYNC_TIMEOUT_SECONDS:-5700}"
GITEE_SYNC_TIMEOUT_SECONDS="${GITEE_SYNC_TIMEOUT_SECONDS:-18840}"
GITEE_TAG_TIMEOUT_SECONDS="${GITEE_TAG_TIMEOUT_SECONDS:-300}"
GITEE_RELEASE_LOOKUP_MAX_TIME="${GITEE_RELEASE_LOOKUP_MAX_TIME:-60}"
GITEE_RELEASE_LOOKUP_RETRIES="${GITEE_RELEASE_LOOKUP_RETRIES:-2}"
GITEE_RELEASE_LOOKUP_RETRY_DELAY="${GITEE_RELEASE_LOOKUP_RETRY_DELAY:-2}"
GITEE_RELEASE_CREATE_MAX_TIME="${GITEE_RELEASE_CREATE_MAX_TIME:-60}"
GITEE_RECONCILE_TIMEOUT_SECONDS="${GITEE_RECONCILE_TIMEOUT_SECONDS:-4920}"
GITEE_RECONCILE_TIMEOUT_SECONDS="${GITEE_RECONCILE_TIMEOUT_SECONDS:-18300}"
GITEE_CHILD_DEADLINE_RESERVE_SECONDS="${GITEE_CHILD_DEADLINE_RESERVE_SECONDS:-5}"
err() {
+285 -17
View File
@@ -381,11 +381,15 @@ func TestReconcileGiteeAssetsRecoversACommittedUploadWithLostResponse(t *testing
scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh"))
distDir := seedGiteeDist(t)
fake := newFakeGiteeRelease(true, false)
fake.listErrorResponsesAfterUpload = 1
fake.listEmptyResponsesAfterUpload = 1
server := httptest.NewServer(fake)
defer server.Close()
cmd := exec.Command("bash", scriptPath)
cmd.Env = giteeAssetEnv(distDir, server.URL, "2")
cmd.Env = append(giteeAssetEnv(distDir, server.URL, "2"),
"GITEE_POST_UPLOAD_VERIFY_ATTEMPTS=2",
)
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("reconcile-gitee-assets.sh error = %v\noutput:\n%s", err, output)
@@ -416,6 +420,142 @@ func TestReconcileGiteeAssetsRecoversACommittedUploadWithLostResponse(t *testing
}
}
func TestReconcileGiteeAssetsDoesNotReplayAnAmbiguousUpload(t *testing.T) {
scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh"))
distDir := seedGiteeDist(t)
fake := newFakeGiteeRelease(true, false)
fake.listEmptyResponsesAfterUpload = 3
server := httptest.NewServer(fake)
defer server.Close()
cmd := exec.Command("bash", scriptPath)
cmd.Env = append(giteeAssetEnv(distDir, server.URL, "2"),
"GITEE_POST_UPLOAD_VERIFY_ATTEMPTS=2",
)
output, err := cmd.CombinedOutput()
if err == nil {
t.Fatalf("ambiguous upload unexpectedly succeeded:\n%s", output)
}
if !strings.Contains(string(output), "refusing to replay POST") {
t.Fatalf("ambiguous upload did not fail closed:\n%s", output)
}
fake.mu.Lock()
defer fake.mu.Unlock()
for _, name := range requiredGiteeAssets {
if got := fake.uploadCalls[name]; got != 1 {
t.Errorf("upload calls for %s = %d, want exactly 1", name, got)
}
}
}
func TestReconcileGiteeAssetsRetriesATransientListOutage(t *testing.T) {
scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh"))
distDir := seedGiteeDist(t)
fake := newFakeGiteeRelease(false, false)
fake.listFailuresRemaining = 2
server := httptest.NewServer(fake)
defer server.Close()
cmd := exec.Command("bash", scriptPath)
cmd.Env = append(giteeAssetEnv(distDir, server.URL, "1"),
"GITEE_LIST_RETRIES=3",
"GITEE_LIST_RETRY_DELAY=1",
"GITEE_LIST_RETRY_WINDOW_SECONDS=4",
)
started := time.Now()
output, err := cmd.CombinedOutput()
elapsed := time.Since(started)
if err != nil {
t.Fatalf("transient-list reconciliation error = %v\noutput:\n%s", err, output)
}
if !strings.Contains(string(output), "Gitee attachment list attempt 2/3 failed; retrying in 1s") {
t.Fatalf("transient list failures were not retried visibly:\n%s", output)
}
if elapsed < 1500*time.Millisecond {
t.Fatalf("transient list retry elapsed = %s, want production-style backoff", elapsed)
}
if !strings.Contains(string(output), "all 8 verified") {
t.Fatalf("transient-list reconciliation did not verify every asset:\n%s", output)
}
fake.mu.Lock()
defer fake.mu.Unlock()
if fake.listFailuresRemaining != 0 {
t.Fatalf("unconsumed list failures = %d, want 0", fake.listFailuresRemaining)
}
for _, name := range requiredGiteeAssets {
if got := fake.uploadCalls[name]; got != 1 {
t.Errorf("upload calls for %s = %d, want 1", name, got)
}
}
}
func TestReconcileGiteeAssetsDoesNotBurstRetriesInTheFinalWindowSecond(t *testing.T) {
scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh"))
distDir := seedGiteeDist(t)
fake := newFakeGiteeRelease(false, false)
fake.listFailuresRemaining = 1000
server := httptest.NewServer(fake)
defer server.Close()
cmd := exec.Command("bash", scriptPath)
cmd.Env = append(giteeAssetEnv(distDir, server.URL, "1"),
"GITEE_LIST_RETRIES=24",
"GITEE_LIST_RETRY_DELAY=1",
"GITEE_LIST_RETRY_WINDOW_SECONDS=1",
)
output, err := cmd.CombinedOutput()
if err == nil {
t.Fatalf("permanent list outage unexpectedly succeeded:\n%s", output)
}
fake.mu.Lock()
defer fake.mu.Unlock()
if fake.listCalls > len(requiredGiteeAssets)+1 {
t.Fatalf("list calls = %d, want at most one per asset plus final verification", fake.listCalls)
}
}
func TestReconcileGiteeAssetsDiscardsPartialOutputFromAMalformedList(t *testing.T) {
scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh"))
distDir := seedGiteeDist(t)
fake := newFakeGiteeRelease(false, false)
existingName := requiredGiteeAssets[0]
existingData, err := os.ReadFile(filepath.Join(distDir, existingName))
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", existingName, err)
}
fake.assets[1] = fakeGiteeAsset{id: 1, name: existingName, data: existingData}
fake.nextID = 2
fake.malformedListResponsesRemaining = 1
server := httptest.NewServer(fake)
defer server.Close()
cmd := exec.Command("bash", scriptPath)
cmd.Env = append(giteeAssetEnv(distDir, server.URL, "1"),
"GITEE_LIST_RETRIES=2",
"GITEE_LIST_RETRY_DELAY=0",
"GITEE_LIST_RETRY_WINDOW_SECONDS=2",
)
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("malformed-list reconciliation error = %v\noutput:\n%s", err, output)
}
if !strings.Contains(string(output), existingName+" already correct on Gitee") {
t.Fatalf("partial failed-list output manufactured a stale or duplicate asset:\n%s", output)
}
if !strings.Contains(string(output), "all 8 verified") {
t.Fatalf("malformed-list reconciliation did not verify every asset:\n%s", output)
}
fake.mu.Lock()
defer fake.mu.Unlock()
if got := fake.uploadCalls[existingName]; got != 0 {
t.Errorf("upload calls for existing %s = %d, want 0", existingName, got)
}
}
func TestReconcileGiteeAssetsDisablesExpectContinueForLargeUploads(t *testing.T) {
scriptPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "reconcile-gitee-assets.sh"))
distDir := seedGiteeDist(t)
@@ -547,7 +687,53 @@ func TestGiteeReleaseWorkflowUsesImmutableTagsAndBoundedRetryBudget(t *testing.T
}
perAssetSeconds := shellDefaultInt(t, reconciler, "GITEE_ASSET_TIMEOUT_SECONDS")
overallSeconds := shellDefaultInt(t, reconciler, "GITEE_OVERALL_TIMEOUT_SECONDS")
finalListSeconds := shellDefaultInt(t, reconciler, "GITEE_LIST_MAX_TIME")
listSeconds := shellDefaultInt(t, reconciler, "GITEE_LIST_MAX_TIME")
listRetries := shellDefaultInt(t, reconciler, "GITEE_LIST_RETRIES")
listRetryDelay := shellDefaultInt(t, reconciler, "GITEE_LIST_RETRY_DELAY")
listRetryWindow := shellDefaultInt(t, reconciler, "GITEE_LIST_RETRY_WINDOW_SECONDS")
uploadSeconds := shellDefaultInt(t, reconciler, "GITEE_UPLOAD_MAX_TIME")
verifySeconds := shellDefaultInt(t, reconciler, "GITEE_VERIFY_MAX_TIME")
postUploadVerifyAttempts := shellDefaultInt(t, reconciler, "GITEE_POST_UPLOAD_VERIFY_ATTEMPTS")
verifyRetryDelay := shellDefaultInt(t, reconciler, "GITEE_VERIFY_RETRY_DELAY")
uploadRetryDelay := shellDefaultInt(t, reconciler, "GITEE_UPLOAD_RETRY_DELAY")
const minimumLargeAssetUploadSeconds = 1200
if uploadSeconds < minimumLargeAssetUploadSeconds {
t.Fatalf(
"Gitee upload deadline = %ds, want at least %ds for near-10 MiB release assets",
uploadSeconds, minimumLargeAssetUploadSeconds,
)
}
const minimumTransientListOutageSeconds = 300
if listRetryWindow < minimumTransientListOutageSeconds {
t.Fatalf(
"Gitee list recovery budget = %ds, want at least %ds for a transient API outage",
listRetryWindow, minimumTransientListOutageSeconds,
)
}
if (listRetries-1)*listRetryDelay < listRetryWindow {
t.Fatalf(
"%d Gitee list attempts with %ds delay cannot span the configured %ds retry window after fast failures",
listRetries, listRetryDelay, listRetryWindow,
)
}
completeListRecoveryBudget := listRetryWindow
oneSlowSuccessBudget := 2*completeListRecoveryBudget + uploadSeconds + verifySeconds
if oneSlowSuccessBudget > perAssetSeconds {
t.Fatalf(
"one complete slow upload budget = %ds, exceeds per-asset deadline %ds",
oneSlowSuccessBudget, perAssetSeconds,
)
}
fastFailureRetryBudget := uploadSeconds + (postUploadVerifyAttempts+3)*listSeconds +
verifySeconds + (postUploadVerifyAttempts-1)*verifyRetryDelay + uploadRetryDelay
fastFailureRetryBudget += 2 * (completeListRecoveryBudget - listSeconds)
if fastFailureRetryBudget > perAssetSeconds {
t.Fatalf(
"fast-failure retry budget = %ds, exceeds per-asset deadline %ds",
fastFailureRetryBudget, perAssetSeconds,
)
}
finalListSeconds := completeListRecoveryBudget
completeAssetBudget := perAssetSeconds*len(requiredGiteeAssets) + finalListSeconds
if completeAssetBudget > overallSeconds {
t.Fatalf(
@@ -562,15 +748,16 @@ func TestGiteeReleaseWorkflowUsesImmutableTagsAndBoundedRetryBudget(t *testing.T
)
}
completeSyncBudget := tagSeconds + lookupSeconds + createSeconds + reconcileSeconds
childDeadlineReserveSeconds := shellDefaultInt(t, syncScript, "GITEE_CHILD_DEADLINE_RESERVE_SECONDS")
completeSyncBudget := tagSeconds + lookupSeconds + createSeconds + reconcileSeconds + childDeadlineReserveSeconds
if completeSyncBudget > syncSeconds {
t.Fatalf(
"complete sync budget = %ds (tag=%d + lookup=%d + create=%d + reconcile=%d), exceeds sync deadline %ds",
completeSyncBudget, tagSeconds, lookupSeconds, createSeconds, reconcileSeconds, syncSeconds,
"complete sync budget = %ds (tag=%d + lookup=%d + create=%d + reconcile=%d + child reserve=%d), exceeds sync deadline %ds",
completeSyncBudget, tagSeconds, lookupSeconds, createSeconds, reconcileSeconds, childDeadlineReserveSeconds, syncSeconds,
)
}
const workflowReserveMinutes = 5
const syncStepReserveMinutes = 4
releasePath := mustAbs(t, filepath.Join("..", "..", ".github", "workflows", "release.yml"))
releaseData, err := os.ReadFile(releasePath)
if err != nil {
@@ -582,21 +769,49 @@ func TestGiteeReleaseWorkflowUsesImmutableTagsAndBoundedRetryBudget(t *testing.T
"mirror-gitee-release:",
[]string{
"name: Check out sealed release source",
"name: Check out trusted release tooling",
"name: Fetch and verify sealed release tag",
"name: Restore finalized distribution files",
"name: Mirror release to Gitee (China)",
},
workflowReserveMinutes,
17,
)
releaseSyncStepSeconds := workflowTimeoutMinutesAfter(
t, string(releaseData), "mirror-gitee-release:", "name: Mirror release to Gitee (China)",
) * 60
if syncSeconds+workflowReserveMinutes*60 > releaseSyncStepSeconds {
if syncSeconds+syncStepReserveMinutes*60 > releaseSyncStepSeconds {
t.Fatalf(
"sync deadline %ds plus reserve exceeds release fallback step %ds",
syncSeconds, releaseSyncStepSeconds,
)
}
assertWorkflowBudget(
t,
string(releaseData),
"repair-channel:",
[]string{
"name: Check out trusted release tooling",
"name: Validate repair version",
"name: Verify immutable release authority",
"name: Check out sealed release source",
"name: Fetch and verify sealed release tag",
"name: Require successful Release workflow delivery",
"name: Download and verify immutable GitHub Release assets",
"name: Mirror release to Gitee (China)",
},
5,
)
repairSyncStepSeconds := workflowTimeoutMinutesAfter(
t, string(releaseData), "repair-channel:", "name: Mirror release to Gitee (China)",
) * 60
if syncSeconds+syncStepReserveMinutes*60 > repairSyncStepSeconds {
t.Fatalf(
"sync deadline %ds plus reserve exceeds repair step %ds",
syncSeconds, repairSyncStepSeconds,
)
}
localBuildPath := mustAbs(t, filepath.Join("..", "..", "scripts", "release", "build-and-publish-gitee.sh"))
localBuildData, err := os.ReadFile(localBuildPath)
if err != nil {
@@ -763,6 +978,9 @@ func giteeAssetEnv(distDir, apiURL, retries string) []string {
"GITEE_RELEASE_ID=1",
"GITEE_CURL_CONNECT_TIMEOUT=2",
"GITEE_CURL_MAX_TIME=2",
"GITEE_LIST_RETRIES=2",
"GITEE_LIST_RETRY_DELAY=0",
"GITEE_LIST_RETRY_WINDOW_SECONDS=2",
"GITEE_UPLOAD_MAX_TIME=2",
"GITEE_UPLOAD_RETRIES="+retries,
"GITEE_UPLOAD_RETRY_DELAY=0",
@@ -779,15 +997,22 @@ type fakeGiteeAsset struct {
}
type fakeGiteeRelease struct {
mu sync.Mutex
nextID int
assets map[int]fakeGiteeAsset
uploadCalls map[string]int
dropFirstResponse bool
droppedResponse bool
failUploads bool
uploadDelay time.Duration
rejectExpectContinue bool
mu sync.Mutex
nextID int
assets map[int]fakeGiteeAsset
uploadCalls map[string]int
dropFirstResponse bool
droppedResponse bool
failUploads bool
uploadDelay time.Duration
rejectExpectContinue bool
listFailuresRemaining int
malformedListResponsesRemaining int
listErrorResponsesRemaining int
listEmptyResponsesRemaining int
listErrorResponsesAfterUpload int
listEmptyResponsesAfterUpload int
listCalls int
}
func newFakeGiteeRelease(dropFirstResponse, failUploads bool) *fakeGiteeRelease {
@@ -832,7 +1057,42 @@ func (f *fakeGiteeRelease) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (f *fakeGiteeRelease) list(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
f.listCalls++
if f.listFailuresRemaining > 0 {
f.listFailuresRemaining--
http.Error(w, "temporary Gitee list outage", http.StatusServiceUnavailable)
return
}
if f.listErrorResponsesRemaining > 0 {
f.listErrorResponsesRemaining--
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{}`)
return
}
if f.listEmptyResponsesRemaining > 0 {
f.listEmptyResponsesRemaining--
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode([]any{})
return
}
baseURL := requestBaseURL(r)
if f.malformedListResponsesRemaining > 0 {
f.malformedListResponsesRemaining--
for _, asset := range f.assets {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode([]any{
map[string]any{
"id": asset.id,
"name": asset.name,
"browser_download_url": fmt.Sprintf("%s/download/%d", baseURL, asset.id),
},
nil,
})
return
}
http.Error(w, "malformed-list fixture requires one asset", http.StatusInternalServerError)
return
}
rows := make([]map[string]any, 0, len(f.assets))
for _, asset := range f.assets {
rows = append(rows, map[string]any{
@@ -884,6 +1144,14 @@ func (f *fakeGiteeRelease) upload(w http.ResponseWriter, r *http.Request) {
id := f.nextID
f.nextID++
f.assets[id] = fakeGiteeAsset{id: id, name: header.Filename, data: data}
if f.listErrorResponsesAfterUpload > 0 {
f.listErrorResponsesRemaining += f.listErrorResponsesAfterUpload
f.listErrorResponsesAfterUpload = 0
}
if f.listEmptyResponsesAfterUpload > 0 {
f.listEmptyResponsesRemaining += f.listEmptyResponsesAfterUpload
f.listEmptyResponsesAfterUpload = 0
}
drop := f.dropFirstResponse && !f.droppedResponse
if drop {
f.droppedResponse = true