fix(kb/ingest): merge blocks on the same row so broken-word pages read
The previous block-based parser left each pymupdf block as its own line. That's fine for prose but wrong for ספר הליקויים's cover and preface, where each word is its own block — you got "ו אנ / מתכבד / ים" instead of "ואנו מתכבדים". Merge blocks whose y-coordinates are within 4 pixels into a single line, right-to-left, joined by spaces. Content pages (where blocks already contain whole paragraphs) are unaffected because a single block on one row stays as one row. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+45
-18
@@ -38,32 +38,59 @@ def _coerce_date(value) -> _dt.date | None:
|
||||
|
||||
|
||||
def _parse_pdf(data: bytes) -> str:
|
||||
"""Extract text from a PDF, preferring block-based parsing.
|
||||
"""Extract text from a PDF, preferring block-based parsing with row merging.
|
||||
|
||||
Plain `get_text("text")` reads runs in raw glyph order which interleaves
|
||||
column labels, page numbers, and body copy for multi-column Hebrew PDFs
|
||||
(e.g. ספר הליקויים). Using `get_text("blocks")` and sorting by vertical
|
||||
position then horizontal position keeps each paragraph intact and
|
||||
preserves reading order top-to-bottom.
|
||||
Plain `get_text("text")` interleaves columns and floating labels for
|
||||
multi-column Hebrew PDFs (e.g. ספר הליקויים) — words come out split to
|
||||
single letters on separate lines. Using `get_text("blocks")` and sorting
|
||||
by vertical position keeps paragraphs intact for well-formatted pages.
|
||||
|
||||
Some PDFs (ספר הליקויים's title/preface pages) emit each word as its
|
||||
own block. Sorting blocks by y-position alone leaves them one-word-per-
|
||||
line. To fix that, we merge blocks that share the same visual row
|
||||
(|Δy| ≤ row_tolerance) into a single line joined by spaces, right-to-
|
||||
left. That makes prose pages readable without affecting well-formed
|
||||
content pages.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
row_tolerance = 4.0 # pixels; blocks within this y-range are the same line
|
||||
|
||||
with fitz.open(stream=data, filetype="pdf") as doc:
|
||||
for page in doc:
|
||||
blocks = page.get_text("blocks") or []
|
||||
# Filter out empty / image-only blocks and sort by (y, x). For RTL
|
||||
# languages, sorting by descending x within the same y keeps the
|
||||
# reading order correct (right-to-left); pymupdf's default block
|
||||
# order already respects this but only per-column — sorting
|
||||
# explicitly makes it robust across oddly-placed floaters.
|
||||
raw = page.get_text("blocks") or []
|
||||
textual = [
|
||||
(b[1], -b[0], b[4])
|
||||
for b in blocks
|
||||
(float(b[0]), float(b[1]), float(b[2]), float(b[3]), b[4])
|
||||
for b in raw
|
||||
if isinstance(b, tuple) and len(b) >= 5 and isinstance(b[4], str) and b[4].strip()
|
||||
]
|
||||
textual.sort() # by y ascending, then x descending (via negation)
|
||||
# Join paragraphs with a blank line so the chunker's header regexes
|
||||
# still see line boundaries.
|
||||
parts.append("\n\n".join(b[2].rstrip() for b in textual))
|
||||
if not textual:
|
||||
continue
|
||||
|
||||
# Sort by y (top→bottom), then by -x so RTL rows read right→left
|
||||
textual.sort(key=lambda t: (t[1], -t[0]))
|
||||
|
||||
# Group into rows by y proximity, then join each row with a space.
|
||||
lines: list[str] = []
|
||||
current_row: list[tuple[float, str]] = []
|
||||
current_y: float | None = None
|
||||
for x0, y0, x1, y1, text in textual:
|
||||
text = text.strip()
|
||||
if not text:
|
||||
continue
|
||||
if current_y is None or abs(y0 - current_y) <= row_tolerance:
|
||||
current_row.append((x0, text))
|
||||
current_y = y0 if current_y is None else current_y
|
||||
else:
|
||||
# Flush previous row
|
||||
current_row.sort(key=lambda p: -p[0]) # RTL
|
||||
lines.append(" ".join(p[1] for p in current_row))
|
||||
current_row = [(x0, text)]
|
||||
current_y = y0
|
||||
if current_row:
|
||||
current_row.sort(key=lambda p: -p[0])
|
||||
lines.append(" ".join(p[1] for p in current_row))
|
||||
|
||||
parts.append("\n\n".join(lines))
|
||||
return "\n\n".join(p for p in parts if p).strip()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user