feat(kb): /kb/source/{id}/pdf streams the original PDF from MinIO
The browse tab in the EspoCRM KnowledgeBase extension needs the original
PDF (with columns, tables, labels) to render layout-heavy sources like
ספר הליקויים readably. The plain-text chunk view loses the structure
because the PDF itself places each word as a separate text block.
New route:
- GET /kb/source/{id}/pdf — resolves the source's `original_path`
(stored as `s3://insurance-kb/processed/<kind>/<file>.pdf`), streams
the bytes from MinIO via the existing kb_s3.fetch helper, and returns
them with Content-Type: application/pdf + inline Content-Disposition.
- 404 when the source doesn't exist.
- 409 when the source was ingested as plain text (e.g. the law from
Wikisource — its original_path ends in .txt, not .pdf).
- Response is chunked (64 KB) to avoid pinning a 2 MB buffer for every
request.
Also select `original_path` in /kb/source/{id}/chunks so the frontend
can decide whether to render the PDF viewer or the chunk list.
Refs Task Master #2
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+87
-1
@@ -12,10 +12,12 @@ import os
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from api.services.kb import search as kb_search
|
||||
from api.services.kb import ingest as kb_ingest
|
||||
from api.services.kb import s3 as kb_s3
|
||||
|
||||
logger = logging.getLogger("shira.kb.public")
|
||||
|
||||
@@ -92,7 +94,7 @@ async def source_chunks(source_id: int, request: Request):
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
source = await conn.fetchrow(
|
||||
"SELECT id, kind, title, identifier, published_at, source_url "
|
||||
"SELECT id, kind, title, identifier, published_at, source_url, original_path "
|
||||
"FROM kb_source WHERE id = $1",
|
||||
source_id,
|
||||
)
|
||||
@@ -111,6 +113,90 @@ async def source_chunks(source_id: int, request: Request):
|
||||
return {"source": src, "chunks": [dict(c) for c in chunks]}
|
||||
|
||||
|
||||
def _parse_s3_uri(uri: str) -> tuple[str, str] | None:
|
||||
"""Split 's3://<bucket>/<key...>' into (bucket, key). None if not s3."""
|
||||
if not uri or not uri.startswith("s3://"):
|
||||
return None
|
||||
rest = uri[len("s3://"):]
|
||||
if "/" not in rest:
|
||||
return None
|
||||
bucket, _, key = rest.partition("/")
|
||||
return bucket, key
|
||||
|
||||
|
||||
@router.get("/source/{source_id}/pdf")
|
||||
async def source_pdf(source_id: int, request: Request):
|
||||
"""Stream the original PDF binary of a source straight from MinIO.
|
||||
|
||||
Used by the EspoCRM KnowledgeBase browse tab to render the PDF inline
|
||||
without exposing shira-hermes or MinIO to the end-user's browser.
|
||||
The EspoCRM EntryPoint is the only caller and proxies this response.
|
||||
|
||||
Returns 404 if the source doesn't exist. Returns 409 if the source
|
||||
has no stored PDF (e.g. the law, which was ingested from plain text
|
||||
and has an `s3://.../file.txt` original_path).
|
||||
"""
|
||||
_verify_auth(request)
|
||||
from api.services.kb.db import get_pool
|
||||
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT title, original_path FROM kb_source WHERE id = $1",
|
||||
source_id,
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="source not found")
|
||||
|
||||
original_path = (row["original_path"] or "").strip()
|
||||
if not original_path.lower().endswith(".pdf"):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="source has no PDF original (stored as non-PDF text)",
|
||||
)
|
||||
|
||||
parsed = _parse_s3_uri(original_path)
|
||||
if not parsed:
|
||||
raise HTTPException(status_code=500, detail="original_path is not an s3 URI")
|
||||
bucket, key = parsed
|
||||
|
||||
# The KB bucket is the only one we serve from here; fetching from an
|
||||
# unexpected bucket would be a bug or a tampered DB row.
|
||||
if bucket != kb_s3._bucket():
|
||||
raise HTTPException(status_code=500, detail="source bucket mismatch")
|
||||
|
||||
try:
|
||||
data = kb_s3.fetch(key)
|
||||
except Exception as e:
|
||||
logger.exception("[kb.pdf] fetch failed for source %s key %s", source_id, key)
|
||||
raise HTTPException(status_code=502, detail=f"storage fetch failed: {e}")
|
||||
|
||||
# Chunk the response so we don't keep the full 2 MB buffer per request
|
||||
# beyond this function's scope.
|
||||
def _chunks(blob: bytes, size: int = 64 * 1024):
|
||||
for i in range(0, len(blob), size):
|
||||
yield blob[i : i + size]
|
||||
|
||||
filename = (row["title"] or f"source-{source_id}").strip() + ".pdf"
|
||||
# Content-Disposition filename must be ASCII-safe on the fallback param,
|
||||
# plus RFC 5987 UTF-8 for the real name.
|
||||
from urllib.parse import quote as _q
|
||||
ascii_fallback = f"source-{source_id}.pdf"
|
||||
disposition = (
|
||||
f"inline; filename=\"{ascii_fallback}\"; filename*=UTF-8''{_q(filename)}"
|
||||
)
|
||||
return StreamingResponse(
|
||||
_chunks(data),
|
||||
media_type="application/pdf",
|
||||
headers={
|
||||
"Content-Disposition": disposition,
|
||||
"Content-Length": str(len(data)),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, max-age=300",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class AskRequest(BaseModel):
|
||||
message: str = Field(..., min_length=1, max_length=2000)
|
||||
conversation_id: str | None = None
|
||||
|
||||
Reference in New Issue
Block a user