shira-hermes/api/routes/legal_aid.py

"""Legal Aid PDF parser endpoint.

Parses a monthly Legal Aid (סיוע משפטי) payment-report PDF and returns the
JSON shape that GreenInvoiceBilling's LegalAidPaymentReportService.ingest()
expects.

v3 (2026-05-03): switched extraction from pdfplumber to Claude Vision via
ai-gateway. The Legal Aid PDF format has multi-line cells, RTL Hebrew column
ordering quirks, and amounts that wrap across rows ("1478.5\\n4" → 1478.54).
pdfplumber's table detection couldn't get past those cleanly. Claude reads
the rendered page image natively and returns structured rows.

Pattern mirrors api/services/ocr.py — render each PDF page with PyMuPDF,
send the PNG to ai-gateway with a JSON-output prompt, then assemble the
result. Auth is X-Api-Key as for the other admin endpoints.
"""

from __future__ import annotations

import base64
import io
import json
import logging
import os
import re
from typing import Any

from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile

logger = logging.getLogger("shira.legal_aid")

router = APIRouter(prefix="/api/legal-aid", tags=["legal-aid"])


def _verify_admin(request: Request) -> None:
    expected = os.environ.get("ADMIN_API_KEY") or os.environ.get("API_KEY", "")
    if not expected:
        raise HTTPException(status_code=503, detail="Admin API key not configured")
    provided = request.headers.get("X-Admin-Key") or request.headers.get("X-Api-Key") or ""
    if provided != expected:
        raise HTTPException(status_code=401, detail="Unauthorized")


_PDF_RENDER_DPI = 180
_PDF_MAX_PAGES = 8  # safety cap; real reports are 1-2 pages


_EXTRACT_PROMPT = """\
אתה מחלץ נתונים מדוח תשלומים חודשי של הסיוע המשפטי (Legal Aid, עברית RTL).

תוציא JSON תקני בלבד (בלי backticks, בלי טקסט נוסף) בסכמה הזו:
{
  "issuerOffice": "Jerusalem" | "TelAviv" | "Haifa" | "BeerSheva" | "Nazareth" | "Center" | "National" | null,
  "reportDate": "YYYY-MM-DD" | null,
  "reportMonth": "YYYY-MM" | null,
  "totalExclVat": number,
  "totalVat": number,
  "totalInclVat": number,
  "lines": [
    {
      "invoiceNumber": "<16 ספרות מתוך 'מספר חשבונית'>",
      "requestNumber": "<7 ספרות מתוך 'מספר בקשה'>",
      "legalAidCaseNumber": "<7 ספרות מתוך 'מספר תיק'>",
      "clientName": "<שם לקוח בעברית, מאוחד אם נשבר לשני שורות>",
      "activityCategory": "<עמודת 'פעולה' — קטגוריית האב, למשל: הודעה, לימוד ענין, התיעצות / פגישה, בקשה, דיון>",
      "activitySubTypeHebrew": "<עמודת 'תת סוג פעולה' — תווית מלאה, למשל: הודעה בהתאם להחלטה, לימוד ענין מלא, התייעצות עם מסמכים>",
      "activityDate": "<עמודת 'תאריך פעילות' בפורמט YYYY-MM-DD>",
      "amountExclVat": <עמודת 'סכום' (ללא מע\"מ) כמספר עשרוני>,
      "vat": <עמודת 'מע\"מ' כמספר עשרוני>,
      "amountInclVat": <עמודת 'סה\"כ' כמספר עשרוני>
    }
  ],
  "travelReimbursementLines": []
}

הנחיות חשובות:
- הטבלה הראשית בעמוד 1 היא "פעולות חייבות במע\"מ" — חלץ ממנה את כל השורות.
- אם תא בטבלה משתרע על שתי שורות (כמו "הודעה בהתאם" + "להחלטה"), אחד אותם לטקסט אחד עם רווח.
- אם סכום נשבר לשורות (כמו "1478.5" ו-"4"), אחד אותם ל-1478.54.
- "מספר חשבונית" הוא תמיד 16 ספרות; "מספר בקשה" ו-"מספר תיק" 7 ספרות כל אחד.
- שדה reportMonth מתוך "עבור חודש: M/YYYY" → "YYYY-MM".
- שדה reportDate מתוך התאריך בכותרת הראשית של הדוח (DD/MM/YYYY → YYYY-MM-DD).
- אם אין שדה — null. אם הטבלה ריקה (כמו טבלת נסיעות בעמוד 2) — אל תכניס את שורותיה ל-lines.
- ה-issuerOffice לפי "מחוז ירושלים" → "Jerusalem" וכו'.

החזר JSON בלבד.
"""


@router.post("/parse-pdf-report")
async def parse_pdf_report(
    request: Request,
    file: UploadFile = File(..., description="Legal Aid monthly payment report PDF"),
    subject: str | None = None,
    debug: bool = Query(default=False),
) -> dict[str, Any]:
    """Parse a Legal Aid monthly payment-report PDF via Claude Vision.

    Auth: X-Api-Key.
    Body: multipart with `file` (the PDF bytes).
    Optional `subject` query: if supplied, used to extract `[סימוכין NNN]` →
        sysref. Manual uploads typically omit this.
    Optional `debug=true` query: also returns the raw model response and per-
        page render info under `_debug` for diagnosing extraction issues.
    """
    _verify_admin(request)

    pdf_bytes = await file.read()
    if not pdf_bytes:
        raise HTTPException(status_code=400, detail="Empty file")
    if pdf_bytes[:4] != b"%PDF":
        raise HTTPException(status_code=400, detail="File is not a PDF")

    try:
        import fitz  # PyMuPDF
    except ImportError as e:
        raise HTTPException(status_code=503, detail=f"PyMuPDF not installed: {e}")

    images_b64: list[tuple[str, str]] = []
    page_count = 0
    try:
        with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
            page_count = min(len(doc), _PDF_MAX_PAGES)
            for i in range(page_count):
                pix = doc[i].get_pixmap(dpi=_PDF_RENDER_DPI)
                png = pix.tobytes("png")
                images_b64.append(("image/png", base64.b64encode(png).decode("ascii")))
    except Exception as e:
        logger.exception("Legal Aid PDF render failed")
        raise HTTPException(status_code=422, detail=f"PDF render error: {e}")

    if not images_b64:
        raise HTTPException(status_code=422, detail="PDF contained no pages")

    raw_response = await _call_vision_for_json(images_b64, _EXTRACT_PROMPT)
    parsed = _parse_json_response(raw_response)

    sysref = None
    if subject:
        m = re.search(r"\[סימוכין\s+(\d+)\]", subject)
        if m:
            sysref = m.group(1)

    out: dict[str, Any] = {
        "subject": subject,
        "sysref": sysref,
        "issuerOffice": parsed.get("issuerOffice"),
        "reportDate": parsed.get("reportDate"),
        "reportMonth": parsed.get("reportMonth"),
        "totalExclVat": _to_float(parsed.get("totalExclVat")),
        "totalVat": _to_float(parsed.get("totalVat")),
        "totalInclVat": _to_float(parsed.get("totalInclVat")),
        "lines": _normalize_lines(parsed.get("lines") or []),
        "travelReimbursementLines": parsed.get("travelReimbursementLines") or [],
    }

    if debug:
        out["_debug"] = {
            "pageCount": page_count,
            "rawResponse": raw_response,
        }

    return out


# ----------------------------------------------------------------------
# AI Gateway / Claude Vision call
# ----------------------------------------------------------------------


async def _call_vision_for_json(
    images_b64: list[tuple[str, str]], prompt: str
) -> str:
    """Send page images to Claude Vision via ai-gateway and return raw text.

    Mirrors the pattern in api/services/ocr.py — uses the OpenAI-compatible
    chat completions endpoint that ai-gateway exposes at /v1.
    """
    try:
        from openai import AsyncOpenAI
    except ImportError as e:
        raise HTTPException(status_code=503, detail=f"openai sdk missing: {e}")

    base_url = os.environ.get("AI_GATEWAY_URL", "http://localhost:3000").rstrip("/") + "/v1"
    api_key = os.environ.get("AI_GATEWAY_API_KEY", "")
    if not api_key:
        raise HTTPException(
            status_code=503,
            detail="AI_GATEWAY_API_KEY is not configured on shira-hermes",
        )

    client = AsyncOpenAI(base_url=base_url, api_key=api_key)

    content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
    for mime, b64 in images_b64:
        content.append(
            {
                "type": "image_url",
                "image_url": {"url": f"data:{mime};base64,{b64}"},
            }
        )

    try:
        response = await client.chat.completions.create(
            model=os.environ.get("CLAUDE_MODEL", "sonnet"),
            messages=[{"role": "user", "content": content}],
            max_tokens=4096,
            temperature=0,
        )
    except Exception as e:
        logger.exception("Legal Aid: ai-gateway call failed")
        raise HTTPException(status_code=502, detail=f"ai-gateway error: {e}")

    return (response.choices[0].message.content or "").strip()


# ----------------------------------------------------------------------
# Response parsing + normalization
# ----------------------------------------------------------------------


def _parse_json_response(text: str) -> dict[str, Any]:
    """Extract a JSON object from the model's raw text.

    Handles a few common Claude failure modes:
      - markdown fences (```json ... ```)
      - trailing commentary after the JSON block
      - smart-quotes in keys/values
    """
    if not text:
        return {}

    # Strip code fences first.
    fence = re.match(r"^\s*```(?:json)?\s*\n(.*?)\n?\s*```\s*$", text, re.DOTALL)
    if fence:
        text = fence.group(1)

    # Take the substring from first `{` to matching last `}`.
    start = text.find("{")
    end = text.rfind("}")
    if start == -1 or end == -1 or end <= start:
        logger.warning("Legal Aid: no JSON object found in model response: %r", text[:300])
        return {}

    candidate = text[start : end + 1]

    try:
        return json.loads(candidate)
    except json.JSONDecodeError:
        # Try one fix-up pass: replace smart quotes, then retry.
        cleaned = (
            candidate.replace("“", '"')
            .replace("”", '"')
            .replace("‘", "'")
            .replace("’", "'")
        )
        try:
            return json.loads(cleaned)
        except json.JSONDecodeError as e:
            logger.warning(
                "Legal Aid: model response is not valid JSON: %s; raw=%r",
                e,
                candidate[:300],
            )
            return {}


def _normalize_lines(lines: list[Any]) -> list[dict[str, Any]]:
    out: list[dict[str, Any]] = []
    for raw in lines:
        if not isinstance(raw, dict):
            continue
        # Drop rows that don't carry the minimum identifying info.
        if not (raw.get("invoiceNumber") or raw.get("legalAidCaseNumber")):
            continue
        out.append(
            {
                "invoiceNumber": _str_or_none(raw.get("invoiceNumber")),
                "requestNumber": _str_or_none(raw.get("requestNumber")),
                "legalAidCaseNumber": _str_or_none(raw.get("legalAidCaseNumber")),
                "clientName": _str_or_none(raw.get("clientName")),
                "activityCategory": _str_or_none(raw.get("activityCategory")),
                "activitySubTypeHebrew": _str_or_none(raw.get("activitySubTypeHebrew")),
                "activityDate": _str_or_none(raw.get("activityDate")),
                "amountExclVat": _to_float(raw.get("amountExclVat")),
                "vat": _to_float(raw.get("vat")),
                "amountInclVat": _to_float(raw.get("amountInclVat")),
            }
        )
    return out


# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------


def _to_float(v: Any) -> float:
    if v is None:
        return 0.0
    if isinstance(v, (int, float)):
        return float(v)
    s = str(v).replace(",", "").replace("₪", "").strip()
    try:
        return float(s)
    except ValueError:
        return 0.0


def _str_or_none(v: Any) -> str | None:
    if v is None:
        return None
    s = str(v).strip()
    return s or None