42 lines
920 B
Go
42 lines
920 B
Go
|
package rag
|
||
|
|
|
||
|
|
import "testing"
|
||
|
|
|
||
|
|
func TestChunkMarkdown_ByHeaders(t *testing.T) {
|
||
|
|
md := `# Title
|
||
|
|
|
||
|
|
## Section 1
|
||
|
|
|
||
|
|
Content of section 1.
|
||
|
|
|
||
|
|
## Section 2
|
||
|
|
|
||
|
|
Content of section 2.`
|
||
|
|
|
||
|
|
chunks := ChunkMarkdown(md, 1000)
|
||
|
|
if len(chunks) < 2 {
|
||
|
|
t.Errorf("expected at least 2 chunks, got %d", len(chunks))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestChunkMarkdown_SmallContent(t *testing.T) {
|
||
|
|
md := "Just a short note with no headers."
|
||
|
|
chunks := ChunkMarkdown(md, 500)
|
||
|
|
if len(chunks) != 1 {
|
||
|
|
t.Errorf("expected 1 chunk, got %d", len(chunks))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestChunkMarkdown_SplitLargeSection(t *testing.T) {
|
||
|
|
md := `## Big Section
|
||
|
|
|
||
|
|
Paragraph one with enough content to make this section reasonably long for testing purposes.
|
||
|
|
|
||
|
|
Paragraph two with more content that should be in a separate chunk when the max size is small.`
|
||
|
|
|
||
|
|
chunks := ChunkMarkdown(md, 80)
|
||
|
|
if len(chunks) < 2 {
|
||
|
|
t.Errorf("expected at least 2 chunks for large section, got %d", len(chunks))
|
||
|
|
}
|
||
|
|
}
|