From ff33114b2cde4e694287470ef2e5c0ab7c9ed0ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E9=9B=A8?= <47820304+PeterGuy326@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:56:53 +0800 Subject: [PATCH] feat(doc): strip server-rejected unsafe chars on markdown write path (#465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doc write boundary only stripped a fixed dangerous-Unicode set, and only on the JSONML path. C0 control characters (except tab/newline), DEL (0x7F), and a few zero-width / line-separator codepoints still reached the server, where RejectControlChars rejects them — so doc create/update failed on content that pasted in such characters (common with LLM-generated or copy-pasted text). - Rename stripDocDangerousUnicode -> stripDocInputUnsafe and extend it to drop C0 controls (except \t and \n) and DEL, matching apiclient.rejectDangerousChars. - Add U+200D, U+2028, U+2029 to the dangerous-Unicode set so it covers the full server-rejected range. - Apply the strip on the markdown write path (doc create/update) and the JSONML node path, not just the JSONML body. - Add unit tests for stripDocInputUnsafe. Ported from dws-wukong (feat: 增加输入安全字符过滤功能). --- internal/helpers/doc.go | 4 +- internal/helpers/doc_jsonml.go | 21 ++++- .../helpers/doc_strip_input_unsafe_test.go | 82 +++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 internal/helpers/doc_strip_input_unsafe_test.go diff --git a/internal/helpers/doc.go b/internal/helpers/doc.go index 29fba2e5..4c68b914 100644 --- a/internal/helpers/doc.go +++ b/internal/helpers/doc.go @@ -362,7 +362,7 @@ func newDocCreateCommand(runner executor.Runner) *cobra.Command { content = stripped } if content != "" { - params["markdown"] = content + params["markdown"] = stripDocInputUnsafe(content) } } } else if format, err := docContentFormat(cmd, "markdown", "jsonml"); err != nil { @@ -473,7 +473,7 @@ func newDocUpdateCommand(runner executor.Runner) *cobra.Command { if sniffJsonMLLike(content) { fmt.Fprintln(cmd.ErrOrStderr(), `warning: 输入内容看起来是 JSONML 结构；若要按 JSONML 解析，请加 --content-format jsonml，否则将按 markdown 解析。`) } - params["markdown"] = content + params["markdown"] = stripDocInputUnsafe(content) addDocIntParam(cmd, params, "index", "index") } return runDocTool(cmd, runner, "doc", "update_document", params) diff --git a/internal/helpers/doc_jsonml.go b/internal/helpers/doc_jsonml.go index b3b8c92b..42fc0856 100644 --- a/internal/helpers/doc_jsonml.go +++ b/internal/helpers/doc_jsonml.go @@ -26,6 +26,7 @@ import ( var docDangerousUnicode = [...]rune{ 0x200B, 0x200C, + 0x200D, 0x200E, 0x200F, 0x202A, @@ -33,6 +34,8 @@ var docDangerousUnicode = [...]rune{ 0x202C, 0x202D, 0x202E, + 0x2028, + 0x2029, 0x2066, 0x2067, 0x2068, @@ -49,8 +52,20 @@ var docDangerousSet = func() map[rune]bool { return m }() -func stripDocDangerousUnicode(s string) string { +// stripDocInputUnsafe removes characters that the server-side RejectControlChars +// validator (mirrored by apiclient.rejectDangerousChars) would reject: +// +// 1. C0 control characters (except tab and newline) and DEL (0x7F) +// 2. Dangerous Unicode (zero-width, Bidi controls, line/paragraph separators, BOM) +// +// It is applied at the write boundary so document content passes server +// validation instead of being rejected. Tab and newline are preserved because +// they are legitimate in document text. +func stripDocInputUnsafe(s string) string { return strings.Map(func(r rune) rune { + if r != '\t' && r != '\n' && (r < 0x20 || r == 0x7F) { + return -1 + } if docDangerousSet[r] { return -1 } @@ -194,7 +209,7 @@ func prepareDocJSONMLBody(cmd *cobra.Command, raw string) (string, error) { if err != nil { return "", fmt.Errorf("marshal normalized jsonml: %w", err) } - return stripDocDangerousUnicode(string(out)), nil + return stripDocInputUnsafe(string(out)), nil } func prepareDocJSONMLNode(cmd *cobra.Command, rawElement string) (string, error) { @@ -247,7 +262,7 @@ func prepareDocJSONMLNode(cmd *cobra.Command, rawElement string) (string, error) if err != nil { return "", fmt.Errorf("marshal normalized jsonml: %w", err) } - return string(out), nil + return stripDocInputUnsafe(string(out)), nil } func docEmitJSONMLFixNotes(cmd *cobra.Command, notes []string) { diff --git a/internal/helpers/doc_strip_input_unsafe_test.go b/internal/helpers/doc_strip_input_unsafe_test.go new file mode 100644 index 00000000..20882727 --- /dev/null +++ b/internal/helpers/doc_strip_input_unsafe_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Alibaba Group +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helpers + +import "testing" + +// TestStripDocInputUnsafe verifies that stripDocInputUnsafe removes exactly the +// characters the server-side RejectControlChars validator rejects (C0 controls +// except tab/newline, DEL, and the dangerous-Unicode set), while leaving all +// legitimate text untouched. Offending codepoints use explicit \u / \x escapes +// so they are unambiguous in source. +func TestStripDocInputUnsafe(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + { + name: "preserves plain text", + in: "正常的文档内容 with ASCII", + want: "正常的文档内容 with ASCII", + }, + { + name: "keeps tab and newline", + in: "标题\n\t正文", + want: "标题\n\t正文", + }, + { + name: "drops C0 controls (null, SOH, CR) and DEL", + in: "正文\x00内容\x01段落\x0d结尾\x7f", + want: "正文内容段落结尾", + }, + { + name: "drops zero-width space/non-joiner/joiner", + in: "正文\u200b内容\u200c段落\u200d结尾", + want: "正文内容段落结尾", + }, + { + name: "drops bidi overrides and isolates", + in: "Bidi\u202a测试\u202e结束\u2066左\u2069右", + want: "Bidi测试结束左右", + }, + { + name: "drops line and paragraph separators", + in: "\u2028\u2029行段", + want: "行段", + }, + { + name: "drops BOM / ZWNBSP", + in: "BOM\ufeff尾", + want: "BOM尾", + }, + { + name: "drops mixed control and dangerous unicode", + in: "混合\x00测试\u200b结尾\x7f", + want: "混合测试结尾", + }, + { + name: "empty string stays empty", + in: "", + want: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := stripDocInputUnsafe(tc.in); got != tc.want { + t.Fatalf("stripDocInputUnsafe(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +}