fix(legal-aid): use Claude Vision for PDF parsing instead of pdfplumber

The real Legal Aid monthly reports have multi-line cells (sub-types like
"הודעה בהתאם להחלטה" wrap to two visual lines), RTL Hebrew column ordering
that flips depending on whether a row contains long Hebrew text, and
amounts that wrap across rows ("1478.5\n4" → 1478.54). pdfplumber's
extract_tables() returned 0 rows on a real report — the v1 parser
silently produced an empty preview.

Switching to the Claude-Vision path (mirrors api/services/ocr.py): render
each PDF page via PyMuPDF, send to ai-gateway with a JSON-output prompt,
parse the structured response. Claude reads RTL Hebrew + multi-line cells
natively and produces the same JSON shape that
LegalAidPaymentReportService.ingest() expects.

`?debug=true` added to return the raw model response for diagnostics
when a future PDF format breaks extraction.

pdfplumber stays in pyproject.toml for now in case we want a non-AI
fallback later, but the runtime path no longer touches it.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

api/routes/legal_aid.py

@@ -1,24 +1,31 @@
 """Legal Aid PDF parser endpoint.
 
-Parses a monthly Legal Aid (סיוע משפטי) payment-report PDF using pdfplumber
+Parses a monthly Legal Aid (סיוע משפטי) payment-report PDF and returns the
-and returns a JSON payload in the shape that GreenInvoiceBilling's
+JSON shape that GreenInvoiceBilling's LegalAidPaymentReportService.ingest()
-LegalAidPaymentReportService.ingest() expects.
+expects.
 
-This is the same parsing logic used by the (currently unimported) n8n workflow
+v3 (2026-05-03): switched extraction from pdfplumber to Claude Vision via
-`workflows/legal-aid-report-ingest.json` in the GreenInvoiceBilling repo —
+ai-gateway. The Legal Aid PDF format has multi-line cells, RTL Hebrew column
-extracted here so the EspoCRM extension can do manual upload + preview without
+ordering quirks, and amounts that wrap across rows ("1478.5\\n4" → 1478.54).
-depending on n8n being live.
+pdfplumber's table detection couldn't get past those cleanly. Claude reads
+the rendered page image natively and returns structured rows.
+
+Pattern mirrors api/services/ocr.py — render each PDF page with PyMuPDF,
+send the PNG to ai-gateway with a JSON-output prompt, then assemble the
+result. Auth is X-Api-Key as for the other admin endpoints.
 """
 
 from __future__ import annotations
 
+import base64
 import io
+import json
 import logging
 import os
 import re
 from typing import Any
 
-from fastapi import APIRouter, File, HTTPException, Request, UploadFile
+from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
 
 logger = logging.getLogger("shira.legal_aid")
 
@@ -34,15 +41,50 @@ def _verify_admin(request: Request) -> None:
         raise HTTPException(status_code=401, detail="Unauthorized")
 
 
-# District name → enum code used in LegalAidPaymentReport.issuerOffice
+_PDF_RENDER_DPI = 180
-_OFFICE_HINTS: list[tuple[str, str]] = [
+_PDF_MAX_PAGES = 8  # safety cap; real reports are 1-2 pages
-    ("ירושלים", "Jerusalem"),
+
-    ("תל אביב", "TelAviv"),
+
-    ("חיפה", "Haifa"),
+_EXTRACT_PROMPT = """\
-    ("באר שבע", "BeerSheva"),
+אתה מחלץ נתונים מדוח תשלומים חודשי של הסיוע המשפטי (Legal Aid, עברית RTL).
-    ("נצרת", "Nazareth"),
+
-    ("מרכז", "Center"),
+תוציא JSON תקני בלבד (בלי backticks, בלי טקסט נוסף) בסכמה הזו:
-]
+{
+  "issuerOffice": "Jerusalem" | "TelAviv" | "Haifa" | "BeerSheva" | "Nazareth" | "Center" | "National" | null,
+  "reportDate": "YYYY-MM-DD" | null,
+  "reportMonth": "YYYY-MM" | null,
+  "totalExclVat": number,
+  "totalVat": number,
+  "totalInclVat": number,
+  "lines": [
+    {
+      "invoiceNumber": "<16 ספרות מתוך 'מספר חשבונית'>",
+      "requestNumber": "<7 ספרות מתוך 'מספר בקשה'>",
+      "legalAidCaseNumber": "<7 ספרות מתוך 'מספר תיק'>",
+      "clientName": "<שם לקוח בעברית, מאוחד אם נשבר לשני שורות>",
+      "activityCategory": "<עמודת 'פעולה' — קטגוריית האב, למשל: הודעה, לימוד ענין, התיעצות / פגישה, בקשה, דיון>",
+      "activitySubTypeHebrew": "<עמודת 'תת סוג פעולה' — תווית מלאה, למשל: הודעה בהתאם להחלטה, לימוד ענין מלא, התייעצות עם מסמכים>",
+      "activityDate": "<עמודת 'תאריך פעילות' בפורמט YYYY-MM-DD>",
+      "amountExclVat": <עמודת 'סכום' (ללא מע\"מ) כמספר עשרוני>,
+      "vat": <עמודת 'מע\"מ' כמספר עשרוני>,
+      "amountInclVat": <עמודת 'סה\"כ' כמספר עשרוני>
+    }
+  ],
+  "travelReimbursementLines": []
+}
+
+הנחיות חשובות:
+- הטבלה הראשית בעמוד 1 היא "פעולות חייבות במע\"מ" — חלץ ממנה את כל השורות.
+- אם תא בטבלה משתרע על שתי שורות (כמו "הודעה בהתאם" + "להחלטה"), אחד אותם לטקסט אחד עם רווח.
+- אם סכום נשבר לשורות (כמו "1478.5" ו-"4"), אחד אותם ל-1478.54.
+- "מספר חשבונית" הוא תמיד 16 ספרות; "מספר בקשה" ו-"מספר תיק" 7 ספרות כל אחד.
+- שדה reportMonth מתוך "עבור חודש: M/YYYY" → "YYYY-MM".
+- שדה reportDate מתוך התאריך בכותרת הראשית של הדוח (DD/MM/YYYY → YYYY-MM-DD).
+- אם אין שדה — null. אם הטבלה ריקה (כמו טבלת נסיעות בעמוד 2) — אל תכניס את שורותיה ל-lines.
+- ה-issuerOffice לפי "מחוז ירושלים" → "Jerusalem" וכו'.
+
+החזר JSON בלבד.
+"""
 
 
 @router.post("/parse-pdf-report")
@@ -50,34 +92,48 @@ async def parse_pdf_report(
     request: Request,
     file: UploadFile = File(..., description="Legal Aid monthly payment report PDF"),
     subject: str | None = None,
+    debug: bool = Query(default=False),
 ) -> dict[str, Any]:
-    """Parse a Legal Aid monthly payment-report PDF.
+    """Parse a Legal Aid monthly payment-report PDF via Claude Vision.
 
-    Auth: X-Api-Key (same as other admin endpoints).
+    Auth: X-Api-Key.
     Body: multipart with `file` (the PDF bytes).
-    Optional `subject` query param: original email subject — when supplied, used
+    Optional `subject` query: if supplied, used to extract `[סימוכין NNN]` →
-        to extract the [סימוכין NNN] sysref. Manual uploads typically omit this.
+        sysref. Manual uploads typically omit this.
-
+    Optional `debug=true` query: also returns the raw model response and per-
-    Returns the same JSON shape that LegalAidPaymentReportService expects:
+        page render info under `_debug` for diagnosing extraction issues.
-        {subject, sysref, issuerOffice, reportDate, reportMonth,
-         totalExclVat, totalVat, totalInclVat, lines:[...], travelReimbursementLines:[]}
     """
     _verify_admin(request)
 
     pdf_bytes = await file.read()
     if not pdf_bytes:
         raise HTTPException(status_code=400, detail="Empty file")
-
+    if pdf_bytes[:4] != b"%PDF":
-    if not (pdf_bytes[:4] == b"%PDF"):
         raise HTTPException(status_code=400, detail="File is not a PDF")
 
     try:
-        import pdfplumber
+        import fitz  # PyMuPDF
     except ImportError as e:
-        raise HTTPException(
+        raise HTTPException(status_code=503, detail=f"PyMuPDF not installed: {e}")
-            status_code=503,
+
-            detail=f"pdfplumber not installed in shira-hermes: {e}",
+    images_b64: list[tuple[str, str]] = []
-        )
+    page_count = 0
+    try:
+        with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
+            page_count = min(len(doc), _PDF_MAX_PAGES)
+            for i in range(page_count):
+                pix = doc[i].get_pixmap(dpi=_PDF_RENDER_DPI)
+                png = pix.tobytes("png")
+                images_b64.append(("image/png", base64.b64encode(png).decode("ascii")))
+    except Exception as e:
+        logger.exception("Legal Aid PDF render failed")
+        raise HTTPException(status_code=422, detail=f"PDF render error: {e}")
+
+    if not images_b64:
+        raise HTTPException(status_code=422, detail="PDF contained no pages")
+
+    raw_response = await _call_vision_for_json(images_b64, _EXTRACT_PROMPT)
+    parsed = _parse_json_response(raw_response)
 
     sysref = None
     if subject:
@@ -85,140 +141,174 @@ async def parse_pdf_report(
         if m:
             sysref = m.group(1)
 
-    issuer_office: str | None = None
-    report_month: str | None = None
-    report_date: str | None = None
-    lines: list[dict[str, Any]] = []
-    total_excl = total_vat = total_incl = 0.0
-    parse_warnings: list[str] = []
-
-    try:
-        with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf:
-            for page in pdf.pages:
-                text = page.extract_text() or ""
-
-                if issuer_office is None:
-                    for hint, code in _OFFICE_HINTS:
-                        if hint in text:
-                            issuer_office = code
-                            break
-
-                if report_month is None:
-                    m = re.search(r"עבור חודש:\s*(\d{1,2})/(\d{4})", text)
-                    if m:
-                        report_month = f"{m.group(2)}-{int(m.group(1)):02d}"
-
-                if report_date is None:
-                    m = re.search(r"(\d{1,2})/(\d{1,2})/(\d{4})", text)
-                    if m:
-                        report_date = (
-                            f"{m.group(3)}-{int(m.group(2)):02d}-{int(m.group(1)):02d}"
-                        )
-
-                for table in page.extract_tables() or []:
-                    if not table or len(table) < 2:
-                        continue
-
-                    headers = [(h or "").strip() for h in table[0]]
-
-                    # Skip the empty travel-reimbursement table on page 2 etc.
-                    if not any("חשבונית" in h for h in headers):
-                        continue
-
-                    for row in table[1:]:
-                        if not row or all((c or "").strip() == "" for c in row):
-                            continue
-
-                        record = _row_to_record(headers, row)
-                        if record is None:
-                            continue
-
-                        total_excl += record.get("amountExclVat") or 0.0
-                        total_vat += record.get("vat") or 0.0
-                        total_incl += record.get("amountInclVat") or 0.0
-                        lines.append(record)
-    except Exception as e:
-        logger.exception("Legal Aid PDF parse failed")
-        raise HTTPException(status_code=422, detail=f"PDF parse error: {e}")
-
     out: dict[str, Any] = {
         "subject": subject,
         "sysref": sysref,
-        "issuerOffice": issuer_office,
+        "issuerOffice": parsed.get("issuerOffice"),
-        "reportDate": report_date,
+        "reportDate": parsed.get("reportDate"),
-        "reportMonth": report_month,
+        "reportMonth": parsed.get("reportMonth"),
-        "totalExclVat": round(total_excl, 2),
+        "totalExclVat": _to_float(parsed.get("totalExclVat")),
-        "totalVat": round(total_vat, 2),
+        "totalVat": _to_float(parsed.get("totalVat")),
-        "totalInclVat": round(total_incl, 2),
+        "totalInclVat": _to_float(parsed.get("totalInclVat")),
-        "lines": lines,
+        "lines": _normalize_lines(parsed.get("lines") or []),
-        "travelReimbursementLines": [],
+        "travelReimbursementLines": parsed.get("travelReimbursementLines") or [],
     }
 
-    if parse_warnings:
+    if debug:
-        out["parseWarnings"] = parse_warnings
+        out["_debug"] = {
+            "pageCount": page_count,
+            "rawResponse": raw_response,
+        }
 
     return out
 
 
-def _row_to_record(headers: list[str], row: list[str | None]) -> dict[str, Any] | None:
+# ----------------------------------------------------------------------
-    """Translate one PDF table row into the structured record the ingest
+# AI Gateway / Claude Vision call
-    service expects. Returns None if the row has nothing to extract.
+# ----------------------------------------------------------------------
+
+
+async def _call_vision_for_json(
+    images_b64: list[tuple[str, str]], prompt: str
+) -> str:
+    """Send page images to Claude Vision via ai-gateway and return raw text.
+
+    Mirrors the pattern in api/services/ocr.py — uses the OpenAI-compatible
+    chat completions endpoint that ai-gateway exposes at /v1.
     """
-    cells = [(c or "").strip() for c in row]
-
-    def col(needle: str) -> str:
-        for i, h in enumerate(headers):
-            if h and needle in h:
-                if i < len(cells):
-                    return cells[i].strip()
-        return ""
-
-    invoice_number = col("חשבונית")
-    request_number = col("בקשה")
-    case_number = col("תיק")
-    client_name = col("לקוח") or col("שם")
-    activity_category_raw = col("פעולה")
-    activity_subtype = col("תת")
-    activity_date_raw = col("תאריך פעילות") or col("תאריך פעיל")
-    amount_excl_raw = col("סכום")
-    vat_raw = col("מע\"מ") or col('מע"מ')
-    amount_incl_raw = col("סה\"כ") or col('סה"כ')
-
-    if not (invoice_number or request_number or case_number):
-        return None
-
-    activity_category = activity_category_raw.split()[0] if activity_category_raw else ""
-    activity_date = _normalize_date(activity_date_raw)
-
-    return {
-        "invoiceNumber": invoice_number,
-        "requestNumber": request_number,
-        "legalAidCaseNumber": case_number,
-        "clientName": client_name,
-        "activityCategory": activity_category,
-        "activitySubTypeHebrew": activity_subtype,
-        "activityDate": activity_date,
-        "amountExclVat": _to_float(amount_excl_raw),
-        "vat": _to_float(vat_raw),
-        "amountInclVat": _to_float(amount_incl_raw),
-        "raw": dict(zip(headers, cells)),
-    }
-
-
-def _normalize_date(s: str) -> str | None:
-    if not s:
-        return None
-    m = re.search(r"(\d{1,2})[./](\d{1,2})[./](\d{4})", s)
-    if not m:
-        return None
-    return f"{m.group(3)}-{int(m.group(2)):02d}-{int(m.group(1)):02d}"
-
-
-def _to_float(s: str) -> float:
-    if not s:
-        return 0.0
-    cleaned = s.replace(",", "").replace("₪", "").replace("₪", "").strip()
     try:
-        return float(cleaned)
+        from openai import AsyncOpenAI
+    except ImportError as e:
+        raise HTTPException(status_code=503, detail=f"openai sdk missing: {e}")
+
+    base_url = os.environ.get("AI_GATEWAY_URL", "http://localhost:3000").rstrip("/") + "/v1"
+    api_key = os.environ.get("AI_GATEWAY_API_KEY", "")
+    if not api_key:
+        raise HTTPException(
+            status_code=503,
+            detail="AI_GATEWAY_API_KEY is not configured on shira-hermes",
+        )
+
+    client = AsyncOpenAI(base_url=base_url, api_key=api_key)
+
+    content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
+    for mime, b64 in images_b64:
+        content.append(
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:{mime};base64,{b64}"},
+            }
+        )
+
+    try:
+        response = await client.chat.completions.create(
+            model=os.environ.get("CLAUDE_MODEL", "sonnet"),
+            messages=[{"role": "user", "content": content}],
+            max_tokens=4096,
+            temperature=0,
+        )
+    except Exception as e:
+        logger.exception("Legal Aid: ai-gateway call failed")
+        raise HTTPException(status_code=502, detail=f"ai-gateway error: {e}")
+
+    return (response.choices[0].message.content or "").strip()
+
+
+# ----------------------------------------------------------------------
+# Response parsing + normalization
+# ----------------------------------------------------------------------
+
+
+def _parse_json_response(text: str) -> dict[str, Any]:
+    """Extract a JSON object from the model's raw text.
+
+    Handles a few common Claude failure modes:
+      - markdown fences (```json ... ```)
+      - trailing commentary after the JSON block
+      - smart-quotes in keys/values
+    """
+    if not text:
+        return {}
+
+    # Strip code fences first.
+    fence = re.match(r"^\s*```(?:json)?\s*\n(.*?)\n?\s*```\s*$", text, re.DOTALL)
+    if fence:
+        text = fence.group(1)
+
+    # Take the substring from first `{` to matching last `}`.
+    start = text.find("{")
+    end = text.rfind("}")
+    if start == -1 or end == -1 or end <= start:
+        logger.warning("Legal Aid: no JSON object found in model response: %r", text[:300])
+        return {}
+
+    candidate = text[start : end + 1]
+
+    try:
+        return json.loads(candidate)
+    except json.JSONDecodeError:
+        # Try one fix-up pass: replace smart quotes, then retry.
+        cleaned = (
+            candidate.replace("“", '"')
+            .replace("”", '"')
+            .replace("‘", "'")
+            .replace("’", "'")
+        )
+        try:
+            return json.loads(cleaned)
+        except json.JSONDecodeError as e:
+            logger.warning(
+                "Legal Aid: model response is not valid JSON: %s; raw=%r",
+                e,
+                candidate[:300],
+            )
+            return {}
+
+
+def _normalize_lines(lines: list[Any]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for raw in lines:
+        if not isinstance(raw, dict):
+            continue
+        # Drop rows that don't carry the minimum identifying info.
+        if not (raw.get("invoiceNumber") or raw.get("legalAidCaseNumber")):
+            continue
+        out.append(
+            {
+                "invoiceNumber": _str_or_none(raw.get("invoiceNumber")),
+                "requestNumber": _str_or_none(raw.get("requestNumber")),
+                "legalAidCaseNumber": _str_or_none(raw.get("legalAidCaseNumber")),
+                "clientName": _str_or_none(raw.get("clientName")),
+                "activityCategory": _str_or_none(raw.get("activityCategory")),
+                "activitySubTypeHebrew": _str_or_none(raw.get("activitySubTypeHebrew")),
+                "activityDate": _str_or_none(raw.get("activityDate")),
+                "amountExclVat": _to_float(raw.get("amountExclVat")),
+                "vat": _to_float(raw.get("vat")),
+                "amountInclVat": _to_float(raw.get("amountInclVat")),
+            }
+        )
+    return out
+
+
+# ----------------------------------------------------------------------
+# Helpers
+# ----------------------------------------------------------------------
+
+
+def _to_float(v: Any) -> float:
+    if v is None:
+        return 0.0
+    if isinstance(v, (int, float)):
+        return float(v)
+    s = str(v).replace(",", "").replace("₪", "").strip()
+    try:
+        return float(s)
     except ValueError:
         return 0.0
+
+
+def _str_or_none(v: Any) -> str | None:
+    if v is None:
+        return None
+    s = str(v).strip()
+    return s or None