diff --git a/api/services/kb/chunker.py b/api/services/kb/chunker.py index d416568..42014ad 100644 --- a/api/services/kb/chunker.py +++ b/api/services/kb/chunker.py @@ -128,6 +128,10 @@ def _as_dicts(chunks: list[Chunk]) -> list[dict]: result: list[dict] = [] for c in chunks: clean, page_in_body = _strip_page_markers(c.content) + # _split_oversized may slice a piece in the middle of a marker + # sentinel, leaving a stray \x00 that PostgreSQL's UTF-8 check + # rejects. Drop any surviving null bytes. + clean = clean.replace("\x00", "") # Prefer the page the caller computed from the chunk's START offset # (the "where does this quote begin" answer). Only fall back to any # marker inside the body if the caller didn't set one.