feat(kb): /kb/source/{id}/pdf streams the original PDF from MinIO
The browse tab in the EspoCRM KnowledgeBase extension needs the original
PDF (with columns, tables, labels) to render layout-heavy sources like
ספר הליקויים readably. The plain-text chunk view loses the structure
because the PDF itself places each word as a separate text block.
New route:
- GET /kb/source/{id}/pdf — resolves the source's `original_path`
(stored as `s3://insurance-kb/processed/<kind>/<file>.pdf`), streams
the bytes from MinIO via the existing kb_s3.fetch helper, and returns
them with Content-Type: application/pdf + inline Content-Disposition.
- 404 when the source doesn't exist.
- 409 when the source was ingested as plain text (e.g. the law from
Wikisource — its original_path ends in .txt, not .pdf).
- Response is chunked (64 KB) to avoid pinning a 2 MB buffer for every
request.
Also select `original_path` in /kb/source/{id}/chunks so the frontend
can decide whether to render the PDF viewer or the chunk list.
Refs Task Master #2
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+87
-1
@@ -12,10 +12,12 @@ import os
|
|||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Request
|
from fastapi import APIRouter, HTTPException, Request
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from api.services.kb import search as kb_search
|
from api.services.kb import search as kb_search
|
||||||
from api.services.kb import ingest as kb_ingest
|
from api.services.kb import ingest as kb_ingest
|
||||||
|
from api.services.kb import s3 as kb_s3
|
||||||
|
|
||||||
logger = logging.getLogger("shira.kb.public")
|
logger = logging.getLogger("shira.kb.public")
|
||||||
|
|
||||||
@@ -92,7 +94,7 @@ async def source_chunks(source_id: int, request: Request):
|
|||||||
pool = await get_pool()
|
pool = await get_pool()
|
||||||
async with pool.acquire() as conn:
|
async with pool.acquire() as conn:
|
||||||
source = await conn.fetchrow(
|
source = await conn.fetchrow(
|
||||||
"SELECT id, kind, title, identifier, published_at, source_url "
|
"SELECT id, kind, title, identifier, published_at, source_url, original_path "
|
||||||
"FROM kb_source WHERE id = $1",
|
"FROM kb_source WHERE id = $1",
|
||||||
source_id,
|
source_id,
|
||||||
)
|
)
|
||||||
@@ -111,6 +113,90 @@ async def source_chunks(source_id: int, request: Request):
|
|||||||
return {"source": src, "chunks": [dict(c) for c in chunks]}
|
return {"source": src, "chunks": [dict(c) for c in chunks]}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_s3_uri(uri: str) -> tuple[str, str] | None:
|
||||||
|
"""Split 's3://<bucket>/<key...>' into (bucket, key). None if not s3."""
|
||||||
|
if not uri or not uri.startswith("s3://"):
|
||||||
|
return None
|
||||||
|
rest = uri[len("s3://"):]
|
||||||
|
if "/" not in rest:
|
||||||
|
return None
|
||||||
|
bucket, _, key = rest.partition("/")
|
||||||
|
return bucket, key
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/source/{source_id}/pdf")
|
||||||
|
async def source_pdf(source_id: int, request: Request):
|
||||||
|
"""Stream the original PDF binary of a source straight from MinIO.
|
||||||
|
|
||||||
|
Used by the EspoCRM KnowledgeBase browse tab to render the PDF inline
|
||||||
|
without exposing shira-hermes or MinIO to the end-user's browser.
|
||||||
|
The EspoCRM EntryPoint is the only caller and proxies this response.
|
||||||
|
|
||||||
|
Returns 404 if the source doesn't exist. Returns 409 if the source
|
||||||
|
has no stored PDF (e.g. the law, which was ingested from plain text
|
||||||
|
and has an `s3://.../file.txt` original_path).
|
||||||
|
"""
|
||||||
|
_verify_auth(request)
|
||||||
|
from api.services.kb.db import get_pool
|
||||||
|
|
||||||
|
pool = await get_pool()
|
||||||
|
async with pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow(
|
||||||
|
"SELECT title, original_path FROM kb_source WHERE id = $1",
|
||||||
|
source_id,
|
||||||
|
)
|
||||||
|
if not row:
|
||||||
|
raise HTTPException(status_code=404, detail="source not found")
|
||||||
|
|
||||||
|
original_path = (row["original_path"] or "").strip()
|
||||||
|
if not original_path.lower().endswith(".pdf"):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="source has no PDF original (stored as non-PDF text)",
|
||||||
|
)
|
||||||
|
|
||||||
|
parsed = _parse_s3_uri(original_path)
|
||||||
|
if not parsed:
|
||||||
|
raise HTTPException(status_code=500, detail="original_path is not an s3 URI")
|
||||||
|
bucket, key = parsed
|
||||||
|
|
||||||
|
# The KB bucket is the only one we serve from here; fetching from an
|
||||||
|
# unexpected bucket would be a bug or a tampered DB row.
|
||||||
|
if bucket != kb_s3._bucket():
|
||||||
|
raise HTTPException(status_code=500, detail="source bucket mismatch")
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = kb_s3.fetch(key)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("[kb.pdf] fetch failed for source %s key %s", source_id, key)
|
||||||
|
raise HTTPException(status_code=502, detail=f"storage fetch failed: {e}")
|
||||||
|
|
||||||
|
# Chunk the response so we don't keep the full 2 MB buffer per request
|
||||||
|
# beyond this function's scope.
|
||||||
|
def _chunks(blob: bytes, size: int = 64 * 1024):
|
||||||
|
for i in range(0, len(blob), size):
|
||||||
|
yield blob[i : i + size]
|
||||||
|
|
||||||
|
filename = (row["title"] or f"source-{source_id}").strip() + ".pdf"
|
||||||
|
# Content-Disposition filename must be ASCII-safe on the fallback param,
|
||||||
|
# plus RFC 5987 UTF-8 for the real name.
|
||||||
|
from urllib.parse import quote as _q
|
||||||
|
ascii_fallback = f"source-{source_id}.pdf"
|
||||||
|
disposition = (
|
||||||
|
f"inline; filename=\"{ascii_fallback}\"; filename*=UTF-8''{_q(filename)}"
|
||||||
|
)
|
||||||
|
return StreamingResponse(
|
||||||
|
_chunks(data),
|
||||||
|
media_type="application/pdf",
|
||||||
|
headers={
|
||||||
|
"Content-Disposition": disposition,
|
||||||
|
"Content-Length": str(len(data)),
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
"Cache-Control": "private, max-age=300",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AskRequest(BaseModel):
|
class AskRequest(BaseModel):
|
||||||
message: str = Field(..., min_length=1, max_length=2000)
|
message: str = Field(..., min_length=1, max_length=2000)
|
||||||
conversation_id: str | None = None
|
conversation_id: str | None = None
|
||||||
|
|||||||
Reference in New Issue
Block a user