From 1ca1cc005063daf12010725e76dfeba1f4b6ed89 Mon Sep 17 00:00:00 2001 From: Chaim Date: Fri, 24 Apr 2026 15:45:41 +0000 Subject: [PATCH] fix(kb/chunker): strip stray \x00 bytes from chunk content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _split_oversized slices at fixed char offsets and can cut a \x00KB_PAGE:N\x00 sentinel in half — the broken prefix/suffix no longer matches _PAGE_MARKER_RE so _strip_page_markers leaves the \x00 byte in place. PostgreSQL's UTF-8 check then rejects the insert with "invalid byte sequence for encoding UTF8: 0x00", and ingest of ספר הליקויים fails on every scan. Belt-and-suspenders: strip all remaining \x00 bytes from clean content before returning. Refs Task Master #2 --- api/services/kb/chunker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/services/kb/chunker.py b/api/services/kb/chunker.py index d416568..42014ad 100644 --- a/api/services/kb/chunker.py +++ b/api/services/kb/chunker.py @@ -128,6 +128,10 @@ def _as_dicts(chunks: list[Chunk]) -> list[dict]: result: list[dict] = [] for c in chunks: clean, page_in_body = _strip_page_markers(c.content) + # _split_oversized may slice a piece in the middle of a marker + # sentinel, leaving a stray \x00 that PostgreSQL's UTF-8 check + # rejects. Drop any surviving null bytes. + clean = clean.replace("\x00", "") # Prefer the page the caller computed from the chunk's START offset # (the "where does this quote begin" answer). Only fall back to any # marker inside the body if the caller didn't set one.