feat(connect): knowledge-base observability + wider text formats (issue #32)
"挂载的本地知识库不生效" was hard to diagnose: a dir of pdf/docx/json indexes nothing, and a per-message retrieval miss was silent — both look identical to the operator. Make the knowledge path observable and a bit more forgiving. - index the .md/.txt FAMILY (.md/.markdown/.mdx/.txt/.text), not just .md/.txt - loadKnowledgeBase logs how many files indexed vs skipped, and the empty-dir error now says only text is indexed (pdf/docx/json are not) - augment logs per message whether it injected N chunks or missed (so a wrong knowledge dir / non-overlapping question is visible, not silent) Tests: isKnowledgeTextExt, markdown-family indexing, clear no-text-files error.
This commit is contained in:
1 parent
1c88dd6a0d
commit
b2fd204e67
2 files changed
+75
-3
No files matched your search
@@ -48,9 +48,24 @@ type knowledgeBase struct {
|
||||
|
||||
// loadKnowledgeBase walks dir for .md/.txt files (hidden dirs skipped) and
|
||||
// indexes them into scored-retrieval chunks.
|
||||
// knowledgeTextExts is the set of plain-text extensions the retriever indexes.
|
||||
var knowledgeTextExts = []string{".md", ".markdown", ".mdx", ".txt", ".text"}
|
||||
|
||||
// isKnowledgeTextExt reports whether ext is an indexable plain-text extension.
|
||||
func isKnowledgeTextExt(ext string) bool {
|
||||
ext = strings.ToLower(ext)
|
||||
for _, e := range knowledgeTextExts {
|
||||
if ext == e {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func loadKnowledgeBase(dir string) (*knowledgeBase, error) {
|
||||
kb := &knowledgeBase{}
|
||||
root := filepath.Clean(dir)
|
||||
indexed, skipped := 0, 0
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
@@ -61,8 +76,8 @@ func loadKnowledgeBase(dir string) (*knowledgeBase, error) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
if ext != ".md" && ext != ".txt" {
|
||||
if !isKnowledgeTextExt(filepath.Ext(path)) {
|
||||
skipped++
|
||||
return nil
|
||||
}
|
||||
if info, ierr := d.Info(); ierr == nil && info.Size() > knowledgeMaxFileBytes {
|
||||
@@ -77,13 +92,21 @@ func loadKnowledgeBase(dir string) (*knowledgeBase, error) {
|
||||
for _, chunk := range splitKnowledgeChunks(string(raw)) {
|
||||
kb.chunks = append(kb.chunks, knowledgeChunk{source: rel, text: chunk, terms: knowledgeTerms(chunk)})
|
||||
}
|
||||
indexed++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Tell the operator when files were skipped — a knowledge dir full of .pdf /
|
||||
// .docx / .json indexes nothing, which otherwise looks like "knowledge has no
|
||||
// effect" (issue #32). Only the .md/.txt text family is indexed.
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "[connect][knowledge] 已索引 %d 个文本文件,跳过 %d 个非文本文件(仅索引 %s)\n",
|
||||
indexed, skipped, strings.Join(knowledgeTextExts, "/"))
|
||||
}
|
||||
if len(kb.chunks) == 0 {
|
||||
return nil, fmt.Errorf("知识目录 %s 下没有可用的 .md/.txt 内容", dir)
|
||||
return nil, fmt.Errorf("知识目录 %s 下没有可用的文本内容(仅索引 %s;pdf/docx/json 等不会被索引)", dir, strings.Join(knowledgeTextExts, "/"))
|
||||
}
|
||||
return kb, nil
|
||||
}
|
||||
@@ -178,6 +201,9 @@ func (kb *knowledgeBase) augment(question string) string {
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
// Observability for "knowledge has no effect" (issue #32): the KB loaded
|
||||
// but nothing in it overlapped this question, so no context was injected.
|
||||
fmt.Fprintf(os.Stderr, "[connect][knowledge] 本条未命中知识库(%d 个片段中无相关内容)\n", len(kb.chunks))
|
||||
return question
|
||||
}
|
||||
sort.Slice(matches, func(a, b int) bool {
|
||||
@@ -200,5 +226,10 @@ func (kb *knowledgeBase) augment(question string) string {
|
||||
}
|
||||
b.WriteString("\n用户问题:")
|
||||
b.WriteString(question)
|
||||
injected := len(matches)
|
||||
if injected > knowledgeTopK {
|
||||
injected = knowledgeTopK
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[connect][knowledge] 本条命中并注入 %d 个知识片段\n", injected)
|
||||
return b.String()
|
||||
}
|
||||
@@ -111,3 +111,44 @@ func TestKnowledgeNilBasePassesThrough(t *testing.T) {
|
||||
t.Fatalf("nil kb should pass through, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsKnowledgeTextExt(t *testing.T) {
|
||||
for _, e := range []string{".md", ".MD", ".markdown", ".mdx", ".txt", ".text"} {
|
||||
if !isKnowledgeTextExt(e) {
|
||||
t.Fatalf("%q should be an indexable text ext", e)
|
||||
}
|
||||
}
|
||||
for _, e := range []string{".pdf", ".docx", ".json", ".png", ""} {
|
||||
if isKnowledgeTextExt(e) {
|
||||
t.Fatalf("%q should NOT be indexable", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeBaseIndexesMarkdownFamily(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "a.markdown"), []byte("# 标题\n\n内容关于考勤制度的说明。"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "b.text"), []byte("纯文本资料一段。"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kb, err := loadKnowledgeBase(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if len(kb.chunks) == 0 {
|
||||
t.Fatal(".markdown/.text should be indexed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeBaseNoTextFilesError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "doc.pdf"), []byte("%PDF-1.7"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := loadKnowledgeBase(dir)
|
||||
if err == nil || !strings.Contains(err.Error(), "没有可用的文本内容") {
|
||||
t.Fatalf("a dir with only non-text files should error clearly, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user