chaim
2628a37c56
Bundling all report pages as 180-DPI PNG base64 in one ai-gateway chat-completions request blew past its body limit and returned 413. A rendered table page is mostly white, so q85 JPEG is several times smaller while keeping the table text crisp enough for Claude Vision. Pairs with the ai-gateway limit bump to 25mb. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
319 lines
12 KiB
Python
319 lines
12 KiB
Python
"""Legal Aid PDF parser endpoint.
|
||
|
||
Parses a monthly Legal Aid (סיוע משפטי) payment-report PDF and returns the
|
||
JSON shape that GreenInvoiceBilling's LegalAidPaymentReportService.ingest()
|
||
expects.
|
||
|
||
v3 (2026-05-03): switched extraction from pdfplumber to Claude Vision via
|
||
ai-gateway. The Legal Aid PDF format has multi-line cells, RTL Hebrew column
|
||
ordering quirks, and amounts that wrap across rows ("1478.5\\n4" → 1478.54).
|
||
pdfplumber's table detection couldn't get past those cleanly. Claude reads
|
||
the rendered page image natively and returns structured rows.
|
||
|
||
Pattern mirrors api/services/ocr.py — render each PDF page with PyMuPDF,
|
||
send the PNG to ai-gateway with a JSON-output prompt, then assemble the
|
||
result. Auth is X-Api-Key as for the other admin endpoints.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import io
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
|
||
|
||
logger = logging.getLogger("shira.legal_aid")
|
||
|
||
router = APIRouter(prefix="/api/legal-aid", tags=["legal-aid"])
|
||
|
||
|
||
def _verify_admin(request: Request) -> None:
|
||
expected = os.environ.get("ADMIN_API_KEY") or os.environ.get("API_KEY", "")
|
||
if not expected:
|
||
raise HTTPException(status_code=503, detail="Admin API key not configured")
|
||
provided = request.headers.get("X-Admin-Key") or request.headers.get("X-Api-Key") or ""
|
||
if provided != expected:
|
||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||
|
||
|
||
_PDF_RENDER_DPI = 180
|
||
_PDF_MAX_PAGES = 8 # safety cap; real reports are 1-2 pages
|
||
|
||
|
||
_EXTRACT_PROMPT = """\
|
||
אתה מחלץ נתונים מדוח תשלומים חודשי של הסיוע המשפטי (Legal Aid, עברית RTL).
|
||
|
||
תוציא JSON תקני בלבד (בלי backticks, בלי טקסט נוסף) בסכמה הזו:
|
||
{
|
||
"issuerOffice": "Jerusalem" | "TelAviv" | "Haifa" | "BeerSheva" | "Nazareth" | "Center" | "National" | null,
|
||
"reportDate": "YYYY-MM-DD" | null,
|
||
"reportMonth": "YYYY-MM" | null,
|
||
"totalExclVat": number,
|
||
"totalVat": number,
|
||
"totalInclVat": number,
|
||
"lines": [
|
||
{
|
||
"invoiceNumber": "<16 ספרות מתוך 'מספר חשבונית'>",
|
||
"requestNumber": "<7 ספרות מתוך 'מספר בקשה'>",
|
||
"legalAidCaseNumber": "<7 ספרות מתוך 'מספר תיק'>",
|
||
"clientName": "<שם לקוח בעברית, מאוחד אם נשבר לשני שורות>",
|
||
"activityCategory": "<עמודת 'פעולה' — קטגוריית האב, למשל: הודעה, לימוד ענין, התיעצות / פגישה, בקשה, דיון>",
|
||
"activitySubTypeHebrew": "<עמודת 'תת סוג פעולה' — תווית מלאה, למשל: הודעה בהתאם להחלטה, לימוד ענין מלא, התייעצות עם מסמכים>",
|
||
"activityDate": "<עמודת 'תאריך פעילות' בפורמט YYYY-MM-DD>",
|
||
"amountExclVat": <עמודת 'סכום' (ללא מע\"מ) כמספר עשרוני>,
|
||
"vat": <עמודת 'מע\"מ' כמספר עשרוני>,
|
||
"amountInclVat": <עמודת 'סה\"כ' כמספר עשרוני>
|
||
}
|
||
],
|
||
"travelReimbursementLines": []
|
||
}
|
||
|
||
הנחיות חשובות:
|
||
- הטבלה הראשית בעמוד 1 היא "פעולות חייבות במע\"מ" — חלץ ממנה את כל השורות.
|
||
- אם תא בטבלה משתרע על שתי שורות (כמו "הודעה בהתאם" + "להחלטה"), אחד אותם לטקסט אחד עם רווח.
|
||
- אם סכום נשבר לשורות (כמו "1478.5" ו-"4"), אחד אותם ל-1478.54.
|
||
- "מספר חשבונית" הוא תמיד 16 ספרות; "מספר בקשה" ו-"מספר תיק" 7 ספרות כל אחד.
|
||
- שדה reportMonth מתוך "עבור חודש: M/YYYY" → "YYYY-MM".
|
||
- שדה reportDate מתוך התאריך בכותרת הראשית של הדוח (DD/MM/YYYY → YYYY-MM-DD).
|
||
- אם אין שדה — null. אם הטבלה ריקה (כמו טבלת נסיעות בעמוד 2) — אל תכניס את שורותיה ל-lines.
|
||
- ה-issuerOffice לפי "מחוז ירושלים" → "Jerusalem" וכו'.
|
||
|
||
החזר JSON בלבד.
|
||
"""
|
||
|
||
|
||
@router.post("/parse-pdf-report")
|
||
async def parse_pdf_report(
|
||
request: Request,
|
||
file: UploadFile = File(..., description="Legal Aid monthly payment report PDF"),
|
||
subject: str | None = None,
|
||
debug: bool = Query(default=False),
|
||
) -> dict[str, Any]:
|
||
"""Parse a Legal Aid monthly payment-report PDF via Claude Vision.
|
||
|
||
Auth: X-Api-Key.
|
||
Body: multipart with `file` (the PDF bytes).
|
||
Optional `subject` query: if supplied, used to extract `[סימוכין NNN]` →
|
||
sysref. Manual uploads typically omit this.
|
||
Optional `debug=true` query: also returns the raw model response and per-
|
||
page render info under `_debug` for diagnosing extraction issues.
|
||
"""
|
||
_verify_admin(request)
|
||
|
||
pdf_bytes = await file.read()
|
||
if not pdf_bytes:
|
||
raise HTTPException(status_code=400, detail="Empty file")
|
||
if pdf_bytes[:4] != b"%PDF":
|
||
raise HTTPException(status_code=400, detail="File is not a PDF")
|
||
|
||
try:
|
||
import fitz # PyMuPDF
|
||
except ImportError as e:
|
||
raise HTTPException(status_code=503, detail=f"PyMuPDF not installed: {e}")
|
||
|
||
images_b64: list[tuple[str, str]] = []
|
||
page_count = 0
|
||
try:
|
||
with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
|
||
page_count = min(len(doc), _PDF_MAX_PAGES)
|
||
for i in range(page_count):
|
||
pix = doc[i].get_pixmap(dpi=_PDF_RENDER_DPI)
|
||
# JPEG (not PNG): a rendered table page is mostly white, and
|
||
# PNG of it is several × larger than a q85 JPEG. Bundling all
|
||
# pages as PNG blew past ai-gateway's body limit (413). JPEG
|
||
# q85 keeps the table text crisp enough for Claude Vision.
|
||
jpg = pix.tobytes("jpeg", jpg_quality=85)
|
||
images_b64.append(("image/jpeg", base64.b64encode(jpg).decode("ascii")))
|
||
except Exception as e:
|
||
logger.exception("Legal Aid PDF render failed")
|
||
raise HTTPException(status_code=422, detail=f"PDF render error: {e}")
|
||
|
||
if not images_b64:
|
||
raise HTTPException(status_code=422, detail="PDF contained no pages")
|
||
|
||
raw_response = await _call_vision_for_json(images_b64, _EXTRACT_PROMPT)
|
||
parsed = _parse_json_response(raw_response)
|
||
|
||
sysref = None
|
||
if subject:
|
||
m = re.search(r"\[סימוכין\s+(\d+)\]", subject)
|
||
if m:
|
||
sysref = m.group(1)
|
||
|
||
out: dict[str, Any] = {
|
||
"subject": subject,
|
||
"sysref": sysref,
|
||
"issuerOffice": parsed.get("issuerOffice"),
|
||
"reportDate": parsed.get("reportDate"),
|
||
"reportMonth": parsed.get("reportMonth"),
|
||
"totalExclVat": _to_float(parsed.get("totalExclVat")),
|
||
"totalVat": _to_float(parsed.get("totalVat")),
|
||
"totalInclVat": _to_float(parsed.get("totalInclVat")),
|
||
"lines": _normalize_lines(parsed.get("lines") or []),
|
||
"travelReimbursementLines": parsed.get("travelReimbursementLines") or [],
|
||
}
|
||
|
||
if debug:
|
||
out["_debug"] = {
|
||
"pageCount": page_count,
|
||
"rawResponse": raw_response,
|
||
}
|
||
|
||
return out
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# AI Gateway / Claude Vision call
|
||
# ----------------------------------------------------------------------
|
||
|
||
|
||
async def _call_vision_for_json(
|
||
images_b64: list[tuple[str, str]], prompt: str
|
||
) -> str:
|
||
"""Send page images to Claude Vision via ai-gateway and return raw text.
|
||
|
||
Mirrors the pattern in api/services/ocr.py — uses the OpenAI-compatible
|
||
chat completions endpoint that ai-gateway exposes at /v1.
|
||
"""
|
||
try:
|
||
from openai import AsyncOpenAI
|
||
except ImportError as e:
|
||
raise HTTPException(status_code=503, detail=f"openai sdk missing: {e}")
|
||
|
||
base_url = os.environ.get("AI_GATEWAY_URL", "http://localhost:3000").rstrip("/") + "/v1"
|
||
api_key = os.environ.get("AI_GATEWAY_API_KEY", "")
|
||
if not api_key:
|
||
raise HTTPException(
|
||
status_code=503,
|
||
detail="AI_GATEWAY_API_KEY is not configured on shira-hermes",
|
||
)
|
||
|
||
client = AsyncOpenAI(base_url=base_url, api_key=api_key)
|
||
|
||
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
||
for mime, b64 in images_b64:
|
||
content.append(
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {"url": f"data:{mime};base64,{b64}"},
|
||
}
|
||
)
|
||
|
||
try:
|
||
response = await client.chat.completions.create(
|
||
model=os.environ.get("CLAUDE_MODEL", "sonnet"),
|
||
messages=[{"role": "user", "content": content}],
|
||
max_tokens=4096,
|
||
temperature=0,
|
||
)
|
||
except Exception as e:
|
||
logger.exception("Legal Aid: ai-gateway call failed")
|
||
raise HTTPException(status_code=502, detail=f"ai-gateway error: {e}")
|
||
|
||
return (response.choices[0].message.content or "").strip()
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# Response parsing + normalization
|
||
# ----------------------------------------------------------------------
|
||
|
||
|
||
def _parse_json_response(text: str) -> dict[str, Any]:
|
||
"""Extract a JSON object from the model's raw text.
|
||
|
||
Handles a few common Claude failure modes:
|
||
- markdown fences (```json ... ```)
|
||
- trailing commentary after the JSON block
|
||
- smart-quotes in keys/values
|
||
"""
|
||
if not text:
|
||
return {}
|
||
|
||
# Strip code fences first.
|
||
fence = re.match(r"^\s*```(?:json)?\s*\n(.*?)\n?\s*```\s*$", text, re.DOTALL)
|
||
if fence:
|
||
text = fence.group(1)
|
||
|
||
# Take the substring from first `{` to matching last `}`.
|
||
start = text.find("{")
|
||
end = text.rfind("}")
|
||
if start == -1 or end == -1 or end <= start:
|
||
logger.warning("Legal Aid: no JSON object found in model response: %r", text[:300])
|
||
return {}
|
||
|
||
candidate = text[start : end + 1]
|
||
|
||
try:
|
||
return json.loads(candidate)
|
||
except json.JSONDecodeError:
|
||
# Try one fix-up pass: replace smart quotes, then retry.
|
||
cleaned = (
|
||
candidate.replace("“", '"')
|
||
.replace("”", '"')
|
||
.replace("‘", "'")
|
||
.replace("’", "'")
|
||
)
|
||
try:
|
||
return json.loads(cleaned)
|
||
except json.JSONDecodeError as e:
|
||
logger.warning(
|
||
"Legal Aid: model response is not valid JSON: %s; raw=%r",
|
||
e,
|
||
candidate[:300],
|
||
)
|
||
return {}
|
||
|
||
|
||
def _normalize_lines(lines: list[Any]) -> list[dict[str, Any]]:
|
||
out: list[dict[str, Any]] = []
|
||
for raw in lines:
|
||
if not isinstance(raw, dict):
|
||
continue
|
||
# Drop rows that don't carry the minimum identifying info.
|
||
if not (raw.get("invoiceNumber") or raw.get("legalAidCaseNumber")):
|
||
continue
|
||
out.append(
|
||
{
|
||
"invoiceNumber": _str_or_none(raw.get("invoiceNumber")),
|
||
"requestNumber": _str_or_none(raw.get("requestNumber")),
|
||
"legalAidCaseNumber": _str_or_none(raw.get("legalAidCaseNumber")),
|
||
"clientName": _str_or_none(raw.get("clientName")),
|
||
"activityCategory": _str_or_none(raw.get("activityCategory")),
|
||
"activitySubTypeHebrew": _str_or_none(raw.get("activitySubTypeHebrew")),
|
||
"activityDate": _str_or_none(raw.get("activityDate")),
|
||
"amountExclVat": _to_float(raw.get("amountExclVat")),
|
||
"vat": _to_float(raw.get("vat")),
|
||
"amountInclVat": _to_float(raw.get("amountInclVat")),
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# Helpers
|
||
# ----------------------------------------------------------------------
|
||
|
||
|
||
def _to_float(v: Any) -> float:
|
||
if v is None:
|
||
return 0.0
|
||
if isinstance(v, (int, float)):
|
||
return float(v)
|
||
s = str(v).replace(",", "").replace("₪", "").strip()
|
||
try:
|
||
return float(s)
|
||
except ValueError:
|
||
return 0.0
|
||
|
||
|
||
def _str_or_none(v: Any) -> str | None:
|
||
if v is None:
|
||
return None
|
||
s = str(v).strip()
|
||
return s or None
|