diff --git a/api/routes/legal_aid.py b/api/routes/legal_aid.py index 09b8724..487471c 100644 --- a/api/routes/legal_aid.py +++ b/api/routes/legal_aid.py @@ -1,24 +1,31 @@ """Legal Aid PDF parser endpoint. -Parses a monthly Legal Aid (סיוע משפטי) payment-report PDF using pdfplumber -and returns a JSON payload in the shape that GreenInvoiceBilling's -LegalAidPaymentReportService.ingest() expects. +Parses a monthly Legal Aid (סיוע משפטי) payment-report PDF and returns the +JSON shape that GreenInvoiceBilling's LegalAidPaymentReportService.ingest() +expects. -This is the same parsing logic used by the (currently unimported) n8n workflow -`workflows/legal-aid-report-ingest.json` in the GreenInvoiceBilling repo — -extracted here so the EspoCRM extension can do manual upload + preview without -depending on n8n being live. +v3 (2026-05-03): switched extraction from pdfplumber to Claude Vision via +ai-gateway. The Legal Aid PDF format has multi-line cells, RTL Hebrew column +ordering quirks, and amounts that wrap across rows ("1478.5\\n4" → 1478.54). +pdfplumber's table detection couldn't get past those cleanly. Claude reads +the rendered page image natively and returns structured rows. + +Pattern mirrors api/services/ocr.py — render each PDF page with PyMuPDF, +send the PNG to ai-gateway with a JSON-output prompt, then assemble the +result. Auth is X-Api-Key as for the other admin endpoints. """ from __future__ import annotations +import base64 import io +import json import logging import os import re from typing import Any -from fastapi import APIRouter, File, HTTPException, Request, UploadFile +from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile logger = logging.getLogger("shira.legal_aid") @@ -34,15 +41,50 @@ def _verify_admin(request: Request) -> None: raise HTTPException(status_code=401, detail="Unauthorized") -# District name → enum code used in LegalAidPaymentReport.issuerOffice -_OFFICE_HINTS: list[tuple[str, str]] = [ - ("ירושלים", "Jerusalem"), - ("תל אביב", "TelAviv"), - ("חיפה", "Haifa"), - ("באר שבע", "BeerSheva"), - ("נצרת", "Nazareth"), - ("מרכז", "Center"), -] +_PDF_RENDER_DPI = 180 +_PDF_MAX_PAGES = 8 # safety cap; real reports are 1-2 pages + + +_EXTRACT_PROMPT = """\ +אתה מחלץ נתונים מדוח תשלומים חודשי של הסיוע המשפטי (Legal Aid, עברית RTL). + +תוציא JSON תקני בלבד (בלי backticks, בלי טקסט נוסף) בסכמה הזו: +{ + "issuerOffice": "Jerusalem" | "TelAviv" | "Haifa" | "BeerSheva" | "Nazareth" | "Center" | "National" | null, + "reportDate": "YYYY-MM-DD" | null, + "reportMonth": "YYYY-MM" | null, + "totalExclVat": number, + "totalVat": number, + "totalInclVat": number, + "lines": [ + { + "invoiceNumber": "<16 ספרות מתוך 'מספר חשבונית'>", + "requestNumber": "<7 ספרות מתוך 'מספר בקשה'>", + "legalAidCaseNumber": "<7 ספרות מתוך 'מספר תיק'>", + "clientName": "<שם לקוח בעברית, מאוחד אם נשבר לשני שורות>", + "activityCategory": "<עמודת 'פעולה' — קטגוריית האב, למשל: הודעה, לימוד ענין, התיעצות / פגישה, בקשה, דיון>", + "activitySubTypeHebrew": "<עמודת 'תת סוג פעולה' — תווית מלאה, למשל: הודעה בהתאם להחלטה, לימוד ענין מלא, התייעצות עם מסמכים>", + "activityDate": "<עמודת 'תאריך פעילות' בפורמט YYYY-MM-DD>", + "amountExclVat": <עמודת 'סכום' (ללא מע\"מ) כמספר עשרוני>, + "vat": <עמודת 'מע\"מ' כמספר עשרוני>, + "amountInclVat": <עמודת 'סה\"כ' כמספר עשרוני> + } + ], + "travelReimbursementLines": [] +} + +הנחיות חשובות: +- הטבלה הראשית בעמוד 1 היא "פעולות חייבות במע\"מ" — חלץ ממנה את כל השורות. +- אם תא בטבלה משתרע על שתי שורות (כמו "הודעה בהתאם" + "להחלטה"), אחד אותם לטקסט אחד עם רווח. +- אם סכום נשבר לשורות (כמו "1478.5" ו-"4"), אחד אותם ל-1478.54. +- "מספר חשבונית" הוא תמיד 16 ספרות; "מספר בקשה" ו-"מספר תיק" 7 ספרות כל אחד. +- שדה reportMonth מתוך "עבור חודש: M/YYYY" → "YYYY-MM". +- שדה reportDate מתוך התאריך בכותרת הראשית של הדוח (DD/MM/YYYY → YYYY-MM-DD). +- אם אין שדה — null. אם הטבלה ריקה (כמו טבלת נסיעות בעמוד 2) — אל תכניס את שורותיה ל-lines. +- ה-issuerOffice לפי "מחוז ירושלים" → "Jerusalem" וכו'. + +החזר JSON בלבד. +""" @router.post("/parse-pdf-report") @@ -50,34 +92,48 @@ async def parse_pdf_report( request: Request, file: UploadFile = File(..., description="Legal Aid monthly payment report PDF"), subject: str | None = None, + debug: bool = Query(default=False), ) -> dict[str, Any]: - """Parse a Legal Aid monthly payment-report PDF. + """Parse a Legal Aid monthly payment-report PDF via Claude Vision. - Auth: X-Api-Key (same as other admin endpoints). + Auth: X-Api-Key. Body: multipart with `file` (the PDF bytes). - Optional `subject` query param: original email subject — when supplied, used - to extract the [סימוכין NNN] sysref. Manual uploads typically omit this. - - Returns the same JSON shape that LegalAidPaymentReportService expects: - {subject, sysref, issuerOffice, reportDate, reportMonth, - totalExclVat, totalVat, totalInclVat, lines:[...], travelReimbursementLines:[]} + Optional `subject` query: if supplied, used to extract `[סימוכין NNN]` → + sysref. Manual uploads typically omit this. + Optional `debug=true` query: also returns the raw model response and per- + page render info under `_debug` for diagnosing extraction issues. """ _verify_admin(request) pdf_bytes = await file.read() if not pdf_bytes: raise HTTPException(status_code=400, detail="Empty file") - - if not (pdf_bytes[:4] == b"%PDF"): + if pdf_bytes[:4] != b"%PDF": raise HTTPException(status_code=400, detail="File is not a PDF") try: - import pdfplumber + import fitz # PyMuPDF except ImportError as e: - raise HTTPException( - status_code=503, - detail=f"pdfplumber not installed in shira-hermes: {e}", - ) + raise HTTPException(status_code=503, detail=f"PyMuPDF not installed: {e}") + + images_b64: list[tuple[str, str]] = [] + page_count = 0 + try: + with fitz.open(stream=pdf_bytes, filetype="pdf") as doc: + page_count = min(len(doc), _PDF_MAX_PAGES) + for i in range(page_count): + pix = doc[i].get_pixmap(dpi=_PDF_RENDER_DPI) + png = pix.tobytes("png") + images_b64.append(("image/png", base64.b64encode(png).decode("ascii"))) + except Exception as e: + logger.exception("Legal Aid PDF render failed") + raise HTTPException(status_code=422, detail=f"PDF render error: {e}") + + if not images_b64: + raise HTTPException(status_code=422, detail="PDF contained no pages") + + raw_response = await _call_vision_for_json(images_b64, _EXTRACT_PROMPT) + parsed = _parse_json_response(raw_response) sysref = None if subject: @@ -85,140 +141,174 @@ async def parse_pdf_report( if m: sysref = m.group(1) - issuer_office: str | None = None - report_month: str | None = None - report_date: str | None = None - lines: list[dict[str, Any]] = [] - total_excl = total_vat = total_incl = 0.0 - parse_warnings: list[str] = [] - - try: - with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf: - for page in pdf.pages: - text = page.extract_text() or "" - - if issuer_office is None: - for hint, code in _OFFICE_HINTS: - if hint in text: - issuer_office = code - break - - if report_month is None: - m = re.search(r"עבור חודש:\s*(\d{1,2})/(\d{4})", text) - if m: - report_month = f"{m.group(2)}-{int(m.group(1)):02d}" - - if report_date is None: - m = re.search(r"(\d{1,2})/(\d{1,2})/(\d{4})", text) - if m: - report_date = ( - f"{m.group(3)}-{int(m.group(2)):02d}-{int(m.group(1)):02d}" - ) - - for table in page.extract_tables() or []: - if not table or len(table) < 2: - continue - - headers = [(h or "").strip() for h in table[0]] - - # Skip the empty travel-reimbursement table on page 2 etc. - if not any("חשבונית" in h for h in headers): - continue - - for row in table[1:]: - if not row or all((c or "").strip() == "" for c in row): - continue - - record = _row_to_record(headers, row) - if record is None: - continue - - total_excl += record.get("amountExclVat") or 0.0 - total_vat += record.get("vat") or 0.0 - total_incl += record.get("amountInclVat") or 0.0 - lines.append(record) - except Exception as e: - logger.exception("Legal Aid PDF parse failed") - raise HTTPException(status_code=422, detail=f"PDF parse error: {e}") - out: dict[str, Any] = { "subject": subject, "sysref": sysref, - "issuerOffice": issuer_office, - "reportDate": report_date, - "reportMonth": report_month, - "totalExclVat": round(total_excl, 2), - "totalVat": round(total_vat, 2), - "totalInclVat": round(total_incl, 2), - "lines": lines, - "travelReimbursementLines": [], + "issuerOffice": parsed.get("issuerOffice"), + "reportDate": parsed.get("reportDate"), + "reportMonth": parsed.get("reportMonth"), + "totalExclVat": _to_float(parsed.get("totalExclVat")), + "totalVat": _to_float(parsed.get("totalVat")), + "totalInclVat": _to_float(parsed.get("totalInclVat")), + "lines": _normalize_lines(parsed.get("lines") or []), + "travelReimbursementLines": parsed.get("travelReimbursementLines") or [], } - if parse_warnings: - out["parseWarnings"] = parse_warnings + if debug: + out["_debug"] = { + "pageCount": page_count, + "rawResponse": raw_response, + } return out -def _row_to_record(headers: list[str], row: list[str | None]) -> dict[str, Any] | None: - """Translate one PDF table row into the structured record the ingest - service expects. Returns None if the row has nothing to extract. +# ---------------------------------------------------------------------- +# AI Gateway / Claude Vision call +# ---------------------------------------------------------------------- + + +async def _call_vision_for_json( + images_b64: list[tuple[str, str]], prompt: str +) -> str: + """Send page images to Claude Vision via ai-gateway and return raw text. + + Mirrors the pattern in api/services/ocr.py — uses the OpenAI-compatible + chat completions endpoint that ai-gateway exposes at /v1. """ - cells = [(c or "").strip() for c in row] - - def col(needle: str) -> str: - for i, h in enumerate(headers): - if h and needle in h: - if i < len(cells): - return cells[i].strip() - return "" - - invoice_number = col("חשבונית") - request_number = col("בקשה") - case_number = col("תיק") - client_name = col("לקוח") or col("שם") - activity_category_raw = col("פעולה") - activity_subtype = col("תת") - activity_date_raw = col("תאריך פעילות") or col("תאריך פעיל") - amount_excl_raw = col("סכום") - vat_raw = col("מע\"מ") or col('מע"מ') - amount_incl_raw = col("סה\"כ") or col('סה"כ') - - if not (invoice_number or request_number or case_number): - return None - - activity_category = activity_category_raw.split()[0] if activity_category_raw else "" - activity_date = _normalize_date(activity_date_raw) - - return { - "invoiceNumber": invoice_number, - "requestNumber": request_number, - "legalAidCaseNumber": case_number, - "clientName": client_name, - "activityCategory": activity_category, - "activitySubTypeHebrew": activity_subtype, - "activityDate": activity_date, - "amountExclVat": _to_float(amount_excl_raw), - "vat": _to_float(vat_raw), - "amountInclVat": _to_float(amount_incl_raw), - "raw": dict(zip(headers, cells)), - } - - -def _normalize_date(s: str) -> str | None: - if not s: - return None - m = re.search(r"(\d{1,2})[./](\d{1,2})[./](\d{4})", s) - if not m: - return None - return f"{m.group(3)}-{int(m.group(2)):02d}-{int(m.group(1)):02d}" - - -def _to_float(s: str) -> float: - if not s: - return 0.0 - cleaned = s.replace(",", "").replace("₪", "").replace("₪", "").strip() try: - return float(cleaned) + from openai import AsyncOpenAI + except ImportError as e: + raise HTTPException(status_code=503, detail=f"openai sdk missing: {e}") + + base_url = os.environ.get("AI_GATEWAY_URL", "http://localhost:3000").rstrip("/") + "/v1" + api_key = os.environ.get("AI_GATEWAY_API_KEY", "") + if not api_key: + raise HTTPException( + status_code=503, + detail="AI_GATEWAY_API_KEY is not configured on shira-hermes", + ) + + client = AsyncOpenAI(base_url=base_url, api_key=api_key) + + content: list[dict[str, Any]] = [{"type": "text", "text": prompt}] + for mime, b64 in images_b64: + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:{mime};base64,{b64}"}, + } + ) + + try: + response = await client.chat.completions.create( + model=os.environ.get("CLAUDE_MODEL", "sonnet"), + messages=[{"role": "user", "content": content}], + max_tokens=4096, + temperature=0, + ) + except Exception as e: + logger.exception("Legal Aid: ai-gateway call failed") + raise HTTPException(status_code=502, detail=f"ai-gateway error: {e}") + + return (response.choices[0].message.content or "").strip() + + +# ---------------------------------------------------------------------- +# Response parsing + normalization +# ---------------------------------------------------------------------- + + +def _parse_json_response(text: str) -> dict[str, Any]: + """Extract a JSON object from the model's raw text. + + Handles a few common Claude failure modes: + - markdown fences (```json ... ```) + - trailing commentary after the JSON block + - smart-quotes in keys/values + """ + if not text: + return {} + + # Strip code fences first. + fence = re.match(r"^\s*```(?:json)?\s*\n(.*?)\n?\s*```\s*$", text, re.DOTALL) + if fence: + text = fence.group(1) + + # Take the substring from first `{` to matching last `}`. + start = text.find("{") + end = text.rfind("}") + if start == -1 or end == -1 or end <= start: + logger.warning("Legal Aid: no JSON object found in model response: %r", text[:300]) + return {} + + candidate = text[start : end + 1] + + try: + return json.loads(candidate) + except json.JSONDecodeError: + # Try one fix-up pass: replace smart quotes, then retry. + cleaned = ( + candidate.replace("“", '"') + .replace("”", '"') + .replace("‘", "'") + .replace("’", "'") + ) + try: + return json.loads(cleaned) + except json.JSONDecodeError as e: + logger.warning( + "Legal Aid: model response is not valid JSON: %s; raw=%r", + e, + candidate[:300], + ) + return {} + + +def _normalize_lines(lines: list[Any]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for raw in lines: + if not isinstance(raw, dict): + continue + # Drop rows that don't carry the minimum identifying info. + if not (raw.get("invoiceNumber") or raw.get("legalAidCaseNumber")): + continue + out.append( + { + "invoiceNumber": _str_or_none(raw.get("invoiceNumber")), + "requestNumber": _str_or_none(raw.get("requestNumber")), + "legalAidCaseNumber": _str_or_none(raw.get("legalAidCaseNumber")), + "clientName": _str_or_none(raw.get("clientName")), + "activityCategory": _str_or_none(raw.get("activityCategory")), + "activitySubTypeHebrew": _str_or_none(raw.get("activitySubTypeHebrew")), + "activityDate": _str_or_none(raw.get("activityDate")), + "amountExclVat": _to_float(raw.get("amountExclVat")), + "vat": _to_float(raw.get("vat")), + "amountInclVat": _to_float(raw.get("amountInclVat")), + } + ) + return out + + +# ---------------------------------------------------------------------- +# Helpers +# ---------------------------------------------------------------------- + + +def _to_float(v: Any) -> float: + if v is None: + return 0.0 + if isinstance(v, (int, float)): + return float(v) + s = str(v).replace(",", "").replace("₪", "").strip() + try: + return float(s) except ValueError: return 0.0 + + +def _str_or_none(v: Any) -> str | None: + if v is None: + return None + s = str(v).strip() + return s or None