fix(kb/ingest): merge blocks on the same row so broken-word pages read
The previous block-based parser left each pymupdf block as its own line. That's fine for prose but wrong for ספר הליקויים's cover and preface, where each word is its own block — you got "ו אנ / מתכבד / ים" instead of "ואנו מתכבדים". Merge blocks whose y-coordinates are within 4 pixels into a single line, right-to-left, joined by spaces. Content pages (where blocks already contain whole paragraphs) are unaffected because a single block on one row stays as one row. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+45
-18
@@ -38,32 +38,59 @@ def _coerce_date(value) -> _dt.date | None:
|
|||||||
|
|
||||||
|
|
||||||
def _parse_pdf(data: bytes) -> str:
|
def _parse_pdf(data: bytes) -> str:
|
||||||
"""Extract text from a PDF, preferring block-based parsing.
|
"""Extract text from a PDF, preferring block-based parsing with row merging.
|
||||||
|
|
||||||
Plain `get_text("text")` reads runs in raw glyph order which interleaves
|
Plain `get_text("text")` interleaves columns and floating labels for
|
||||||
column labels, page numbers, and body copy for multi-column Hebrew PDFs
|
multi-column Hebrew PDFs (e.g. ספר הליקויים) — words come out split to
|
||||||
(e.g. ספר הליקויים). Using `get_text("blocks")` and sorting by vertical
|
single letters on separate lines. Using `get_text("blocks")` and sorting
|
||||||
position then horizontal position keeps each paragraph intact and
|
by vertical position keeps paragraphs intact for well-formatted pages.
|
||||||
preserves reading order top-to-bottom.
|
|
||||||
|
Some PDFs (ספר הליקויים's title/preface pages) emit each word as its
|
||||||
|
own block. Sorting blocks by y-position alone leaves them one-word-per-
|
||||||
|
line. To fix that, we merge blocks that share the same visual row
|
||||||
|
(|Δy| ≤ row_tolerance) into a single line joined by spaces, right-to-
|
||||||
|
left. That makes prose pages readable without affecting well-formed
|
||||||
|
content pages.
|
||||||
"""
|
"""
|
||||||
parts: list[str] = []
|
parts: list[str] = []
|
||||||
|
row_tolerance = 4.0 # pixels; blocks within this y-range are the same line
|
||||||
|
|
||||||
with fitz.open(stream=data, filetype="pdf") as doc:
|
with fitz.open(stream=data, filetype="pdf") as doc:
|
||||||
for page in doc:
|
for page in doc:
|
||||||
blocks = page.get_text("blocks") or []
|
raw = page.get_text("blocks") or []
|
||||||
# Filter out empty / image-only blocks and sort by (y, x). For RTL
|
|
||||||
# languages, sorting by descending x within the same y keeps the
|
|
||||||
# reading order correct (right-to-left); pymupdf's default block
|
|
||||||
# order already respects this but only per-column — sorting
|
|
||||||
# explicitly makes it robust across oddly-placed floaters.
|
|
||||||
textual = [
|
textual = [
|
||||||
(b[1], -b[0], b[4])
|
(float(b[0]), float(b[1]), float(b[2]), float(b[3]), b[4])
|
||||||
for b in blocks
|
for b in raw
|
||||||
if isinstance(b, tuple) and len(b) >= 5 and isinstance(b[4], str) and b[4].strip()
|
if isinstance(b, tuple) and len(b) >= 5 and isinstance(b[4], str) and b[4].strip()
|
||||||
]
|
]
|
||||||
textual.sort() # by y ascending, then x descending (via negation)
|
if not textual:
|
||||||
# Join paragraphs with a blank line so the chunker's header regexes
|
continue
|
||||||
# still see line boundaries.
|
|
||||||
parts.append("\n\n".join(b[2].rstrip() for b in textual))
|
# Sort by y (top→bottom), then by -x so RTL rows read right→left
|
||||||
|
textual.sort(key=lambda t: (t[1], -t[0]))
|
||||||
|
|
||||||
|
# Group into rows by y proximity, then join each row with a space.
|
||||||
|
lines: list[str] = []
|
||||||
|
current_row: list[tuple[float, str]] = []
|
||||||
|
current_y: float | None = None
|
||||||
|
for x0, y0, x1, y1, text in textual:
|
||||||
|
text = text.strip()
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
if current_y is None or abs(y0 - current_y) <= row_tolerance:
|
||||||
|
current_row.append((x0, text))
|
||||||
|
current_y = y0 if current_y is None else current_y
|
||||||
|
else:
|
||||||
|
# Flush previous row
|
||||||
|
current_row.sort(key=lambda p: -p[0]) # RTL
|
||||||
|
lines.append(" ".join(p[1] for p in current_row))
|
||||||
|
current_row = [(x0, text)]
|
||||||
|
current_y = y0
|
||||||
|
if current_row:
|
||||||
|
current_row.sort(key=lambda p: -p[0])
|
||||||
|
lines.append(" ".join(p[1] for p in current_row))
|
||||||
|
|
||||||
|
parts.append("\n\n".join(lines))
|
||||||
return "\n\n".join(p for p in parts if p).strip()
|
return "\n\n".join(p for p in parts if p).strip()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user