chaim
5ddacbe573
Adds a MinIO-backed ingest flow: upload to s3://insurance-kb/inbox/<kind>/ via MinIO Console or any S3 client, then POST /admin/kb/scan-inbox to process all pending files. Successful ingests move to processed/<kind>/, failures move to failed/<kind>/ with the error stored as S3 metadata. - api/services/kb/s3.py: boto3 client, list_inbox/fetch/move helpers. - api/routes/admin_kb.py: GET /admin/kb/inbox, POST /admin/kb/scan-inbox. - The kind is derived from the folder path; title/identifier default to the filename stem and can be refined by re-ingesting with metadata. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
126 lines
4.1 KiB
Python
126 lines
4.1 KiB
Python
"""Admin endpoints for managing the Israeli National Insurance KB."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from typing import Literal
|
|
|
|
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
|
|
|
from api.services.kb import ingest as kb_ingest
|
|
from api.services.kb import s3 as kb_s3
|
|
|
|
logger = logging.getLogger("shira.admin.kb")
|
|
|
|
router = APIRouter(prefix="/admin/kb", tags=["admin-kb"])
|
|
|
|
|
|
def _verify_admin(request: Request) -> None:
|
|
expected = os.environ.get("ADMIN_API_KEY") or os.environ.get("API_KEY", "")
|
|
if not expected:
|
|
raise HTTPException(status_code=503, detail="Admin API key not configured")
|
|
provided = request.headers.get("X-Admin-Key") or request.headers.get("X-Api-Key") or ""
|
|
if provided != expected:
|
|
raise HTTPException(status_code=401, detail="Unauthorized")
|
|
|
|
|
|
@router.post("/ingest")
|
|
async def ingest(
|
|
request: Request,
|
|
kind: Literal["law", "regulation", "circular"] = Form(...),
|
|
title: str = Form(...),
|
|
identifier: str | None = Form(None),
|
|
published_at: str | None = Form(None),
|
|
effective_at: str | None = Form(None),
|
|
source_url: str | None = Form(None),
|
|
original_path: str | None = Form(None),
|
|
file: UploadFile = File(...),
|
|
):
|
|
_verify_admin(request)
|
|
data = await file.read()
|
|
if not data:
|
|
raise HTTPException(status_code=400, detail="Empty file")
|
|
try:
|
|
result = await kb_ingest.ingest_source(
|
|
kind=kind,
|
|
title=title,
|
|
identifier=identifier,
|
|
content=data,
|
|
filename=file.filename or "upload",
|
|
published_at=published_at,
|
|
effective_at=effective_at,
|
|
source_url=source_url,
|
|
original_path=original_path,
|
|
)
|
|
except kb_ingest.IngestError as e:
|
|
raise HTTPException(status_code=422, detail=str(e))
|
|
except Exception as e:
|
|
logger.exception("[admin.kb] ingest failed")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
return result
|
|
|
|
|
|
@router.get("/stats")
|
|
async def stats(request: Request):
|
|
_verify_admin(request)
|
|
return await kb_ingest.stats()
|
|
|
|
|
|
@router.get("/inbox")
|
|
async def list_inbox(request: Request):
|
|
_verify_admin(request)
|
|
return {"items": kb_s3.list_inbox()}
|
|
|
|
|
|
def _derive_metadata(filename: str, kind: str) -> dict:
|
|
"""Best-effort metadata from the filename stem. User can override later."""
|
|
stem = filename.rsplit("/", 1)[-1]
|
|
for ext in (".pdf", ".docx", ".txt", ".md"):
|
|
if stem.lower().endswith(ext):
|
|
stem = stem[: -len(ext)]
|
|
break
|
|
# Title is just the stem; identifier left empty unless obvious.
|
|
return {"title": stem, "identifier": None}
|
|
|
|
|
|
@router.post("/scan-inbox")
|
|
async def scan_inbox(request: Request):
|
|
_verify_admin(request)
|
|
items = kb_s3.list_inbox()
|
|
results = []
|
|
for item in items:
|
|
src_key = item["key"]
|
|
kind = item["kind"]
|
|
filename = item["filename"]
|
|
meta = _derive_metadata(filename, kind)
|
|
try:
|
|
data = kb_s3.fetch(src_key)
|
|
result = await kb_ingest.ingest_source(
|
|
kind=kind,
|
|
title=meta["title"],
|
|
identifier=meta["identifier"],
|
|
content=data,
|
|
filename=filename,
|
|
original_path=f"s3://{kb_s3._bucket()}/processed/{kind}/{filename}",
|
|
)
|
|
dst = kb_s3.mark_processed(src_key, filename, kind)
|
|
results.append({
|
|
"key": src_key,
|
|
"status": "processed",
|
|
"destination": dst,
|
|
**result,
|
|
})
|
|
except Exception as e:
|
|
logger.exception("[scan-inbox] failed for %s", src_key)
|
|
dst = kb_s3.mark_failed(src_key, filename, kind, str(e))
|
|
results.append({
|
|
"key": src_key,
|
|
"status": "failed",
|
|
"destination": dst,
|
|
"error": str(e),
|
|
})
|
|
return {"processed": sum(1 for r in results if r["status"] == "processed"),
|
|
"failed": sum(1 for r in results if r["status"] == "failed"),
|
|
"items": results}
|