diff --git a/api/services/kb/ingest.py b/api/services/kb/ingest.py index 288b555..fb30709 100644 --- a/api/services/kb/ingest.py +++ b/api/services/kb/ingest.py @@ -38,32 +38,59 @@ def _coerce_date(value) -> _dt.date | None: def _parse_pdf(data: bytes) -> str: - """Extract text from a PDF, preferring block-based parsing. + """Extract text from a PDF, preferring block-based parsing with row merging. - Plain `get_text("text")` reads runs in raw glyph order which interleaves - column labels, page numbers, and body copy for multi-column Hebrew PDFs - (e.g. ספר הליקויים). Using `get_text("blocks")` and sorting by vertical - position then horizontal position keeps each paragraph intact and - preserves reading order top-to-bottom. + Plain `get_text("text")` interleaves columns and floating labels for + multi-column Hebrew PDFs (e.g. ספר הליקויים) — words come out split to + single letters on separate lines. Using `get_text("blocks")` and sorting + by vertical position keeps paragraphs intact for well-formatted pages. + + Some PDFs (ספר הליקויים's title/preface pages) emit each word as its + own block. Sorting blocks by y-position alone leaves them one-word-per- + line. To fix that, we merge blocks that share the same visual row + (|Δy| ≤ row_tolerance) into a single line joined by spaces, right-to- + left. That makes prose pages readable without affecting well-formed + content pages. """ parts: list[str] = [] + row_tolerance = 4.0 # pixels; blocks within this y-range are the same line + with fitz.open(stream=data, filetype="pdf") as doc: for page in doc: - blocks = page.get_text("blocks") or [] - # Filter out empty / image-only blocks and sort by (y, x). For RTL - # languages, sorting by descending x within the same y keeps the - # reading order correct (right-to-left); pymupdf's default block - # order already respects this but only per-column — sorting - # explicitly makes it robust across oddly-placed floaters. + raw = page.get_text("blocks") or [] textual = [ - (b[1], -b[0], b[4]) - for b in blocks + (float(b[0]), float(b[1]), float(b[2]), float(b[3]), b[4]) + for b in raw if isinstance(b, tuple) and len(b) >= 5 and isinstance(b[4], str) and b[4].strip() ] - textual.sort() # by y ascending, then x descending (via negation) - # Join paragraphs with a blank line so the chunker's header regexes - # still see line boundaries. - parts.append("\n\n".join(b[2].rstrip() for b in textual)) + if not textual: + continue + + # Sort by y (top→bottom), then by -x so RTL rows read right→left + textual.sort(key=lambda t: (t[1], -t[0])) + + # Group into rows by y proximity, then join each row with a space. + lines: list[str] = [] + current_row: list[tuple[float, str]] = [] + current_y: float | None = None + for x0, y0, x1, y1, text in textual: + text = text.strip() + if not text: + continue + if current_y is None or abs(y0 - current_y) <= row_tolerance: + current_row.append((x0, text)) + current_y = y0 if current_y is None else current_y + else: + # Flush previous row + current_row.sort(key=lambda p: -p[0]) # RTL + lines.append(" ".join(p[1] for p in current_row)) + current_row = [(x0, text)] + current_y = y0 + if current_row: + current_row.sort(key=lambda p: -p[0]) + lines.append(" ".join(p[1] for p in current_row)) + + parts.append("\n\n".join(lines)) return "\n\n".join(p for p in parts if p).strip()