From 748c8f10c91f5b197b5e1ed89a27685af16e5e76 Mon Sep 17 00:00:00 2001 From: Chaim Date: Fri, 24 Apr 2026 11:51:07 +0000 Subject: [PATCH] feat(kb): /kb/source/{id}/pdf streams the original PDF from MinIO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browse tab in the EspoCRM KnowledgeBase extension needs the original PDF (with columns, tables, labels) to render layout-heavy sources like ספר הליקויים readably. The plain-text chunk view loses the structure because the PDF itself places each word as a separate text block. New route: - GET /kb/source/{id}/pdf — resolves the source's `original_path` (stored as `s3://insurance-kb/processed//.pdf`), streams the bytes from MinIO via the existing kb_s3.fetch helper, and returns them with Content-Type: application/pdf + inline Content-Disposition. - 404 when the source doesn't exist. - 409 when the source was ingested as plain text (e.g. the law from Wikisource — its original_path ends in .txt, not .pdf). - Response is chunked (64 KB) to avoid pinning a 2 MB buffer for every request. Also select `original_path` in /kb/source/{id}/chunks so the frontend can decide whether to render the PDF viewer or the chunk list. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) --- api/routes/kb_public.py | 88 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/api/routes/kb_public.py b/api/routes/kb_public.py index f62ecbb..f3573e6 100644 --- a/api/routes/kb_public.py +++ b/api/routes/kb_public.py @@ -12,10 +12,12 @@ import os from typing import Literal from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from api.services.kb import search as kb_search from api.services.kb import ingest as kb_ingest +from api.services.kb import s3 as kb_s3 logger = logging.getLogger("shira.kb.public") @@ -92,7 +94,7 @@ async def source_chunks(source_id: int, request: Request): pool = await get_pool() async with pool.acquire() as conn: source = await conn.fetchrow( - "SELECT id, kind, title, identifier, published_at, source_url " + "SELECT id, kind, title, identifier, published_at, source_url, original_path " "FROM kb_source WHERE id = $1", source_id, ) @@ -111,6 +113,90 @@ async def source_chunks(source_id: int, request: Request): return {"source": src, "chunks": [dict(c) for c in chunks]} +def _parse_s3_uri(uri: str) -> tuple[str, str] | None: + """Split 's3:///' into (bucket, key). None if not s3.""" + if not uri or not uri.startswith("s3://"): + return None + rest = uri[len("s3://"):] + if "/" not in rest: + return None + bucket, _, key = rest.partition("/") + return bucket, key + + +@router.get("/source/{source_id}/pdf") +async def source_pdf(source_id: int, request: Request): + """Stream the original PDF binary of a source straight from MinIO. + + Used by the EspoCRM KnowledgeBase browse tab to render the PDF inline + without exposing shira-hermes or MinIO to the end-user's browser. + The EspoCRM EntryPoint is the only caller and proxies this response. + + Returns 404 if the source doesn't exist. Returns 409 if the source + has no stored PDF (e.g. the law, which was ingested from plain text + and has an `s3://.../file.txt` original_path). + """ + _verify_auth(request) + from api.services.kb.db import get_pool + + pool = await get_pool() + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT title, original_path FROM kb_source WHERE id = $1", + source_id, + ) + if not row: + raise HTTPException(status_code=404, detail="source not found") + + original_path = (row["original_path"] or "").strip() + if not original_path.lower().endswith(".pdf"): + raise HTTPException( + status_code=409, + detail="source has no PDF original (stored as non-PDF text)", + ) + + parsed = _parse_s3_uri(original_path) + if not parsed: + raise HTTPException(status_code=500, detail="original_path is not an s3 URI") + bucket, key = parsed + + # The KB bucket is the only one we serve from here; fetching from an + # unexpected bucket would be a bug or a tampered DB row. + if bucket != kb_s3._bucket(): + raise HTTPException(status_code=500, detail="source bucket mismatch") + + try: + data = kb_s3.fetch(key) + except Exception as e: + logger.exception("[kb.pdf] fetch failed for source %s key %s", source_id, key) + raise HTTPException(status_code=502, detail=f"storage fetch failed: {e}") + + # Chunk the response so we don't keep the full 2 MB buffer per request + # beyond this function's scope. + def _chunks(blob: bytes, size: int = 64 * 1024): + for i in range(0, len(blob), size): + yield blob[i : i + size] + + filename = (row["title"] or f"source-{source_id}").strip() + ".pdf" + # Content-Disposition filename must be ASCII-safe on the fallback param, + # plus RFC 5987 UTF-8 for the real name. + from urllib.parse import quote as _q + ascii_fallback = f"source-{source_id}.pdf" + disposition = ( + f"inline; filename=\"{ascii_fallback}\"; filename*=UTF-8''{_q(filename)}" + ) + return StreamingResponse( + _chunks(data), + media_type="application/pdf", + headers={ + "Content-Disposition": disposition, + "Content-Length": str(len(data)), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, max-age=300", + }, + ) + + class AskRequest(BaseModel): message: str = Field(..., min_length=1, max_length=2000) conversation_id: str | None = None