feat(kb): /kb/source/{id}/pdf streams the original PDF from MinIO

The browse tab in the EspoCRM KnowledgeBase extension needs the original
PDF (with columns, tables, labels) to render layout-heavy sources like
ספר הליקויים readably. The plain-text chunk view loses the structure
because the PDF itself places each word as a separate text block.

New route:
- GET /kb/source/{id}/pdf — resolves the source's `original_path`
  (stored as `s3://insurance-kb/processed/<kind>/<file>.pdf`), streams
  the bytes from MinIO via the existing kb_s3.fetch helper, and returns
  them with Content-Type: application/pdf + inline Content-Disposition.
- 404 when the source doesn't exist.
- 409 when the source was ingested as plain text (e.g. the law from
  Wikisource — its original_path ends in .txt, not .pdf).
- Response is chunked (64 KB) to avoid pinning a 2 MB buffer for every
  request.

Also select `original_path` in /kb/source/{id}/chunks so the frontend
can decide whether to render the PDF viewer or the chunk list.

Refs Task Master #2

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-24 11:51:07 +00:00
parent 93e4d4ab64
commit 748c8f10c9
+87 -1
View File
@@ -12,10 +12,12 @@ import os
from typing import Literal
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from api.services.kb import search as kb_search
from api.services.kb import ingest as kb_ingest
from api.services.kb import s3 as kb_s3
logger = logging.getLogger("shira.kb.public")
@@ -92,7 +94,7 @@ async def source_chunks(source_id: int, request: Request):
pool = await get_pool()
async with pool.acquire() as conn:
source = await conn.fetchrow(
"SELECT id, kind, title, identifier, published_at, source_url "
"SELECT id, kind, title, identifier, published_at, source_url, original_path "
"FROM kb_source WHERE id = $1",
source_id,
)
@@ -111,6 +113,90 @@ async def source_chunks(source_id: int, request: Request):
return {"source": src, "chunks": [dict(c) for c in chunks]}
def _parse_s3_uri(uri: str) -> tuple[str, str] | None:
"""Split 's3://<bucket>/<key...>' into (bucket, key). None if not s3."""
if not uri or not uri.startswith("s3://"):
return None
rest = uri[len("s3://"):]
if "/" not in rest:
return None
bucket, _, key = rest.partition("/")
return bucket, key
@router.get("/source/{source_id}/pdf")
async def source_pdf(source_id: int, request: Request):
"""Stream the original PDF binary of a source straight from MinIO.
Used by the EspoCRM KnowledgeBase browse tab to render the PDF inline
without exposing shira-hermes or MinIO to the end-user's browser.
The EspoCRM EntryPoint is the only caller and proxies this response.
Returns 404 if the source doesn't exist. Returns 409 if the source
has no stored PDF (e.g. the law, which was ingested from plain text
and has an `s3://.../file.txt` original_path).
"""
_verify_auth(request)
from api.services.kb.db import get_pool
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow(
"SELECT title, original_path FROM kb_source WHERE id = $1",
source_id,
)
if not row:
raise HTTPException(status_code=404, detail="source not found")
original_path = (row["original_path"] or "").strip()
if not original_path.lower().endswith(".pdf"):
raise HTTPException(
status_code=409,
detail="source has no PDF original (stored as non-PDF text)",
)
parsed = _parse_s3_uri(original_path)
if not parsed:
raise HTTPException(status_code=500, detail="original_path is not an s3 URI")
bucket, key = parsed
# The KB bucket is the only one we serve from here; fetching from an
# unexpected bucket would be a bug or a tampered DB row.
if bucket != kb_s3._bucket():
raise HTTPException(status_code=500, detail="source bucket mismatch")
try:
data = kb_s3.fetch(key)
except Exception as e:
logger.exception("[kb.pdf] fetch failed for source %s key %s", source_id, key)
raise HTTPException(status_code=502, detail=f"storage fetch failed: {e}")
# Chunk the response so we don't keep the full 2 MB buffer per request
# beyond this function's scope.
def _chunks(blob: bytes, size: int = 64 * 1024):
for i in range(0, len(blob), size):
yield blob[i : i + size]
filename = (row["title"] or f"source-{source_id}").strip() + ".pdf"
# Content-Disposition filename must be ASCII-safe on the fallback param,
# plus RFC 5987 UTF-8 for the real name.
from urllib.parse import quote as _q
ascii_fallback = f"source-{source_id}.pdf"
disposition = (
f"inline; filename=\"{ascii_fallback}\"; filename*=UTF-8''{_q(filename)}"
)
return StreamingResponse(
_chunks(data),
media_type="application/pdf",
headers={
"Content-Disposition": disposition,
"Content-Length": str(len(data)),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, max-age=300",
},
)
class AskRequest(BaseModel):
message: str = Field(..., min_length=1, max_length=2000)
conversation_id: str | None = None