feat(connect): knowledge-base observability + wider text formats (issue #32)

"挂载的本地知识库不生效" was hard to diagnose: a dir of pdf/docx/json indexes
nothing, and a per-message retrieval miss was silent — both look identical to
the operator. Make the knowledge path observable and a bit more forgiving.

- index the .md/.txt FAMILY (.md/.markdown/.mdx/.txt/.text), not just .md/.txt
- loadKnowledgeBase logs how many files indexed vs skipped, and the empty-dir
  error now says only text is indexed (pdf/docx/json are not)
- augment logs per message whether it injected N chunks or missed (so a wrong
  knowledge dir / non-overlapping question is visible, not silent)

Tests: isKnowledgeTextExt, markdown-family indexing, clear no-text-files error.
This commit is contained in:
修雨 committed 2026-06-15 22:27:14 +08:00
1 parent 1c88dd6a0d
commit b2fd204e67
2 files changed
+75 -3

No files matched your search

+34 -3
View File
@@ -48,9 +48,24 @@ type knowledgeBase struct {
// loadKnowledgeBase walks dir for .md/.txt files (hidden dirs skipped) and
// indexes them into scored-retrieval chunks.
// knowledgeTextExts is the set of plain-text extensions the retriever indexes.
var knowledgeTextExts = []string{".md", ".markdown", ".mdx", ".txt", ".text"}
// isKnowledgeTextExt reports whether ext is an indexable plain-text extension.
func isKnowledgeTextExt(ext string) bool {
ext = strings.ToLower(ext)
for _, e := range knowledgeTextExts {
if ext == e {
return true
}
}
return false
}
func loadKnowledgeBase(dir string) (*knowledgeBase, error) {
kb := &knowledgeBase{}
root := filepath.Clean(dir)
indexed, skipped := 0, 0
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
@@ -61,8 +76,8 @@ func loadKnowledgeBase(dir string) (*knowledgeBase, error) {
}
return nil
}
ext := strings.ToLower(filepath.Ext(path))
if ext != ".md" && ext != ".txt" {
if !isKnowledgeTextExt(filepath.Ext(path)) {
skipped++
return nil
}
if info, ierr := d.Info(); ierr == nil && info.Size() > knowledgeMaxFileBytes {
@@ -77,13 +92,21 @@ func loadKnowledgeBase(dir string) (*knowledgeBase, error) {
for _, chunk := range splitKnowledgeChunks(string(raw)) {
kb.chunks = append(kb.chunks, knowledgeChunk{source: rel, text: chunk, terms: knowledgeTerms(chunk)})
}
indexed++
return nil
})
if err != nil {
return nil, err
}
// Tell the operator when files were skipped — a knowledge dir full of .pdf /
// .docx / .json indexes nothing, which otherwise looks like "knowledge has no
// effect" (issue #32). Only the .md/.txt text family is indexed.
if skipped > 0 {
fmt.Fprintf(os.Stderr, "[connect][knowledge] 已索引 %d 个文本文件,跳过 %d 个非文本文件(仅索引 %s)\n",
indexed, skipped, strings.Join(knowledgeTextExts, "/"))
}
if len(kb.chunks) == 0 {
return nil, fmt.Errorf("知识目录 %s 下没有可用的 .md/.txt 内容", dir)
return nil, fmt.Errorf("知识目录 %s 下没有可用的文本内容(仅索引 %s;pdf/docx/json 等不会被索引)", dir, strings.Join(knowledgeTextExts, "/"))
}
return kb, nil
}
@@ -178,6 +201,9 @@ func (kb *knowledgeBase) augment(question string) string {
}
}
if len(matches) == 0 {
// Observability for "knowledge has no effect" (issue #32): the KB loaded
// but nothing in it overlapped this question, so no context was injected.
fmt.Fprintf(os.Stderr, "[connect][knowledge] 本条未命中知识库(%d 个片段中无相关内容)\n", len(kb.chunks))
return question
}
sort.Slice(matches, func(a, b int) bool {
@@ -200,5 +226,10 @@ func (kb *knowledgeBase) augment(question string) string {
}
b.WriteString("\n用户问题:")
b.WriteString(question)
injected := len(matches)
if injected > knowledgeTopK {
injected = knowledgeTopK
}
fmt.Fprintf(os.Stderr, "[connect][knowledge] 本条命中并注入 %d 个知识片段\n", injected)
return b.String()
}
@@ -111,3 +111,44 @@ func TestKnowledgeNilBasePassesThrough(t *testing.T) {
t.Fatalf("nil kb should pass through, got %q", got)
}
}
func TestIsKnowledgeTextExt(t *testing.T) {
for _, e := range []string{".md", ".MD", ".markdown", ".mdx", ".txt", ".text"} {
if !isKnowledgeTextExt(e) {
t.Fatalf("%q should be an indexable text ext", e)
}
}
for _, e := range []string{".pdf", ".docx", ".json", ".png", ""} {
if isKnowledgeTextExt(e) {
t.Fatalf("%q should NOT be indexable", e)
}
}
}
func TestKnowledgeBaseIndexesMarkdownFamily(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "a.markdown"), []byte("# 标题\n\n内容关于考勤制度的说明。"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "b.text"), []byte("纯文本资料一段。"), 0o644); err != nil {
t.Fatal(err)
}
kb, err := loadKnowledgeBase(dir)
if err != nil {
t.Fatalf("load: %v", err)
}
if len(kb.chunks) == 0 {
t.Fatal(".markdown/.text should be indexed")
}
}
func TestKnowledgeBaseNoTextFilesError(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "doc.pdf"), []byte("%PDF-1.7"), 0o644); err != nil {
t.Fatal(err)
}
_, err := loadKnowledgeBase(dir)
if err == nil || !strings.Contains(err.Error(), "没有可用的文本内容") {
t.Fatalf("a dir with only non-text files should error clearly, got %v", err)
}
}