shira-hermes/api/routes/admin_kb.py

"""Admin endpoints for managing the Israeli National Insurance KB."""

from __future__ import annotations

import asyncio
import logging
import os
from typing import Literal

from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile

from api.services.kb import ingest as kb_ingest
from api.services.kb import ingest_jobs as kb_jobs
from api.services.kb import s3 as kb_s3
from api.services.kb import topics as kb_topics
from api.services.kb.db import get_pool

# Arbitrary non-zero constant used as a pg_try_advisory_lock key. Any int fits;
# this one is a stable hash of "kb-scan-inbox" so it can't collide with any
# other advisory lock chosen the same way for a different purpose.
_SCAN_LOCK_KEY = 0x4B42_5343  # "KBSC" in ASCII

logger = logging.getLogger("shira.admin.kb")

router = APIRouter(prefix="/admin/kb", tags=["admin-kb"])


def _verify_admin(request: Request) -> None:
    expected = os.environ.get("ADMIN_API_KEY") or os.environ.get("API_KEY", "")
    if not expected:
        raise HTTPException(status_code=503, detail="Admin API key not configured")
    provided = request.headers.get("X-Admin-Key") or request.headers.get("X-Api-Key") or ""
    if provided != expected:
        raise HTTPException(status_code=401, detail="Unauthorized")


@router.post("/ingest")
async def ingest(
    request: Request,
    kind: Literal["law", "regulation", "circular", "caselaw"] = Form(...),
    title: str = Form(...),
    identifier: str | None = Form(None),
    published_at: str | None = Form(None),
    effective_at: str | None = Form(None),
    source_url: str | None = Form(None),
    original_path: str | None = Form(None),
    file: UploadFile = File(...),
):
    _verify_admin(request)
    data = await file.read()
    if not data:
        raise HTTPException(status_code=400, detail="Empty file")
    try:
        result = await kb_ingest.ingest_source(
            kind=kind,
            title=title,
            identifier=identifier,
            content=data,
            filename=file.filename or "upload",
            published_at=published_at,
            effective_at=effective_at,
            source_url=source_url,
            original_path=original_path,
        )
    except kb_ingest.IngestError as e:
        raise HTTPException(status_code=422, detail=str(e))
    except Exception as e:
        logger.exception("[admin.kb] ingest failed")
        raise HTTPException(status_code=500, detail=str(e))
    return result


@router.get("/stats")
async def stats(request: Request):
    _verify_admin(request)
    return await kb_ingest.stats()


@router.get("/inbox")
async def list_inbox(request: Request):
    _verify_admin(request)
    return {"items": kb_s3.list_inbox()}


def _derive_metadata(filename: str, kind: str) -> dict:
    """Best-effort metadata from the filename stem. User can override later."""
    stem = filename.rsplit("/", 1)[-1]
    for ext in (".pdf", ".docx", ".txt", ".md"):
        if stem.lower().endswith(ext):
            stem = stem[: -len(ext)]
            break
    return {
        "title": stem,
        "identifier": None,
        "published_at": None,
        "effective_at": None,
        "source_url": None,
    }


def _sidecar_metadata(item_key: str) -> dict:
    """Fetch the optional `<key>.meta.json` sidecar and return parsed fields.

    The sidecar is expected to be a JSON object with any of:
      {title, identifier, published_at, effective_at, source_url}

    Missing or unreadable sidecars return {}.
    """
    import json
    sidecar_key = item_key + ".meta.json"
    try:
        data = kb_s3.fetch(sidecar_key)
    except Exception:
        return {}
    try:
        parsed = json.loads(data.decode("utf-8"))
    except Exception as e:
        logger.warning("[admin.kb] bad sidecar %s: %s", sidecar_key, e)
        return {}
    # Accept only whitelisted keys — ignore typos / extra fields.
    allowed = {"title", "identifier", "published_at", "effective_at", "source_url"}
    return {k: v for k, v in parsed.items() if k in allowed and v}


@router.post("/requeue-failed")
async def requeue_failed(request: Request):
    """Move everything under failed/<kind>/ back to inbox/<kind>/ for retry.

    The sidecar `<name>.meta.json` (if present) is moved alongside.
    """
    _verify_admin(request)
    items = kb_s3.list_failed()
    moved = []
    for item in items:
        src = item["key"]
        dst = f"inbox/{item['kind']}/{item['filename']}"
        try:
            kb_s3.move(src, dst)
            # Best-effort sidecar move.
            try:
                kb_s3.move(src + ".meta.json", dst + ".meta.json")
            except Exception:
                pass
            moved.append({"from": src, "to": dst})
        except Exception as e:
            moved.append({"from": src, "error": str(e)})
    return {"requeued": len([m for m in moved if "to" in m]), "items": moved}


@router.post("/scan-inbox")
async def scan_inbox(request: Request):
    """Scan the inbox and ingest each pending file.

    Protected by a session-level Postgres advisory lock so two concurrent
    callers (e.g. overlapping n8n cron runs after a worker outage) can't both
    process the same file and create duplicate sources. A caller that fails
    to acquire the lock gets `skipped: lock held`.
    """
    _verify_admin(request)
    pool = await get_pool()
    async with pool.acquire() as conn:
        got_lock = await conn.fetchval(
            "SELECT pg_try_advisory_lock($1)", _SCAN_LOCK_KEY
        )
        if not got_lock:
            logger.info("[scan-inbox] skipped: another scan is in progress")
            return {"processed": 0, "failed": 0, "skipped": True, "items": []}
        try:
            return await _do_scan()
        finally:
            await conn.execute(
                "SELECT pg_advisory_unlock($1)", _SCAN_LOCK_KEY
            )


async def _do_scan() -> dict:
    items = kb_s3.list_inbox()
    results = []
    for item in items:
        src_key = item["key"]
        kind = item["kind"]
        filename = item["filename"]
        # Merge: filename-derived defaults < sidecar overrides.
        meta = _derive_metadata(filename, kind)
        meta.update(_sidecar_metadata(src_key))
        try:
            data = kb_s3.fetch(src_key)
            result = await kb_ingest.ingest_source(
                kind=kind,
                title=meta["title"],
                identifier=meta.get("identifier"),
                content=data,
                filename=filename,
                published_at=meta.get("published_at"),
                effective_at=meta.get("effective_at"),
                source_url=meta.get("source_url"),
                original_path=f"s3://{kb_s3._bucket()}/processed/{kind}/{filename}",
            )
            dst = kb_s3.mark_processed(src_key, filename, kind)
            results.append({
                "key": src_key,
                "status": "processed",
                "destination": dst,
                **result,
            })
        except Exception as e:
            logger.exception("[scan-inbox] failed for %s", src_key)
            dst = kb_s3.mark_failed(src_key, filename, kind, str(e))
            results.append({
                "key": src_key,
                "status": "failed",
                "destination": dst,
                "error": str(e),
            })
    return {"processed": sum(1 for r in results if r["status"] == "processed"),
            "failed": sum(1 for r in results if r["status"] == "failed"),
            "items": results}


# ── Phase 2: async upload + job tracking (Task #13) ────────────────────────

# Bytes; cap upload size so a stray 500MB scan doesn't OOM the API container.
_MAX_UPLOAD_BYTES = 50 * 1024 * 1024


@router.post("/upload")
async def upload(
    request: Request,
    kind: Literal["law", "regulation", "circular", "caselaw"] = Form(...),
    topic_id: int = Form(...),
    title: str | None = Form(None),
    identifier: str | None = Form(None),
    published_at: str | None = Form(None),
    effective_at: str | None = Form(None),
    source_url: str | None = Form(None),
    file: UploadFile = File(...),
):
    """Async upload: queue a kb_ingest_job, return immediately with job_id.

    The browser polls `/admin/kb/jobs/{id}` until it sees a terminal state.
    Errors during parse/embed are recorded on the job row, not bubbled here.
    """
    _verify_admin(request)

    data = await file.read()
    if not data:
        raise HTTPException(status_code=400, detail="Empty file")
    if len(data) > _MAX_UPLOAD_BYTES:
        raise HTTPException(
            status_code=413,
            detail=f"File too large ({len(data)} bytes; max {_MAX_UPLOAD_BYTES})",
        )

    # Validate the topic exists and is active. Using resolve_topic_id
    # mirrors what /kb/ask and /kb/search already do.
    try:
        topic = await kb_topics.get_topic(topic_id)
    except Exception:
        topic = None
    if not topic or not topic.get("is_active"):
        raise HTTPException(status_code=400, detail=f"Unknown or inactive topic_id={topic_id}")

    requested_by = (
        request.headers.get("X-User-Name")
        or request.headers.get("X-Forwarded-User")
        or None
    )

    metadata = {
        "title": (title or "").strip() or None,
        "identifier": (identifier or "").strip() or None,
        "published_at": published_at or None,
        "effective_at": effective_at or None,
        "source_url": source_url or None,
    }

    try:
        job_id = await kb_jobs.create_job(
            topic_id=topic_id,
            topic_slug=topic["slug"],
            kind=kind,
            original_filename=file.filename or "upload",
            file_bytes=data,
            metadata=metadata,
            requested_by_user=requested_by,
        )
    except Exception as e:
        logger.exception("[admin.kb.upload] failed to create job")
        raise HTTPException(status_code=500, detail=f"Failed to queue job: {e}")

    # Fire-and-forget: don't await. The HTTP response returns now; the
    # background task transitions the job through processing → done|failed.
    asyncio.create_task(kb_jobs.process_job(job_id))

    return {"job_id": job_id, "status": "queued"}


@router.get("/jobs")
async def list_ingest_jobs(
    request: Request,
    topic_id: int | None = None,
    status: str | None = None,
    limit: int = 50,
):
    _verify_admin(request)
    if status and status not in ("queued", "processing", "done", "failed"):
        raise HTTPException(status_code=400, detail=f"Invalid status: {status}")
    items = await kb_jobs.list_jobs(topic_id=topic_id, status=status, limit=limit)
    return {"items": items}


@router.get("/jobs/{job_id}")
async def get_ingest_job(request: Request, job_id: int):
    _verify_admin(request)
    job = await kb_jobs.get_job(job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job not found")
    return job