feat(kb): backend for v0.8.0 — tool kind + labels + AI classifier + bulk upload
Phase 6 of the multi-topic KB refactor (backend half). Subsumes the original Task #19 design discussion (subject vs labels). EspoCRM extension UI ships separately. Migrations (4 new, all idempotent, transaction-wrapped): 004_kind_tool — adds 'tool' to kb_source/kb_ingest_job CHECK 005_source_description — kb_source.description + ai_classified_at 006_labels — kb_label + kb_source_label (M:N), GRANTs 007_ingest_classifier — awaiting_review status + processing_stage + ai_suggestions JSONB + batch_id Two-phase ingest (api/services/kb/ingest_jobs.py): queued → processing(stage=classifying) → awaiting_review ← user reviews AI output → processing(stage=embedding) ← after user commits → done | failed process_classify_stage() pulls the file, runs parse_first_pages (3-page text extract), calls classifier.classify (Claude tool-call via ai-gateway, 45s timeout, 5-concurrent semaphore, fail-soft on any error → empty suggestions), writes ai_suggestions JSONB, transitions to awaiting_review. commit_job() resolves user-confirmed labels (existing by slug, new ones via slugify+ON CONFLICT DO NOTHING), transitions to processing(embedding), schedules process_embed_stage. process_embed_stage() runs the legacy ingest_source path with user-edited metadata + summary as kb_source.description, then apply_to_source for labels. discard_job() removes a file from a batch (status=failed, S3 deleted). list_jobs_by_batch() — single-roundtrip review-screen load. Labels (NEW api/services/kb/labels.py): Hebrew → ASCII slugify (letter-by-letter map, no external dep); CRUD; lookup_or_create_batch (race-safe); apply_to_source + replace_for_source maintain usage_count; merge race-safely (UPDATE assignments + ON CONFLICT DO NOTHING). Classifier (NEW api/services/kb/classifier.py): AsyncOpenAI to ai-gateway, Sonnet, OpenAI-style tool calling for guaranteed JSON shape, Hebrew system prompt with 5 kind values, citation format examples, "do not invent" rule, summary length bounds 80-200 chars. Routes (api/routes/admin_kb.py — 8 new on top of existing 11): POST /admin/kb/upload-batch (multi-file, AI classify) GET /admin/kb/batch/{batch_id} (review-screen load) POST /admin/kb/jobs/{id}/commit (user confirms metadata) POST /admin/kb/jobs/{id}/discard (user removes from batch) GET /admin/kb/labels (typeahead) POST /admin/kb/labels (explicit create) POST /admin/kb/labels/{id}/merge (admin cleanup) DELETE /admin/kb/labels/{id} (only if usage=0) Public /kb/search gains optional label_id filter. admin_sources.list_sources LEFT JOINs labels into each row's output. update_source accepts label_ids to replace the set. End-to-end smoke test passed: synthetic Hebrew circular → upload-batch → background classify → awaiting_review with kind, title, subject, summary, identifier, published_at all populated correctly. Refs Task Master #20 (espocrm-extensions/KnowledgeBase v0.8.0) Subsumes: Task #19 (labels for sub-topics) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+278
-6
@@ -13,6 +13,7 @@ from pydantic import BaseModel, Field
|
||||
from api.services.kb import admin_sources as kb_admin_sources
|
||||
from api.services.kb import ingest as kb_ingest
|
||||
from api.services.kb import ingest_jobs as kb_jobs
|
||||
from api.services.kb import labels as kb_labels
|
||||
from api.services.kb import s3 as kb_s3
|
||||
from api.services.kb import topics as kb_topics
|
||||
from api.services.kb.db import get_pool
|
||||
@@ -39,7 +40,7 @@ def _verify_admin(request: Request) -> None:
|
||||
@router.post("/ingest")
|
||||
async def ingest(
|
||||
request: Request,
|
||||
kind: Literal["law", "regulation", "circular", "caselaw"] = Form(...),
|
||||
kind: Literal["law", "regulation", "circular", "caselaw", "tool"] = Form(...),
|
||||
title: str = Form(...),
|
||||
identifier: str | None = Form(None),
|
||||
published_at: str | None = Form(None),
|
||||
@@ -228,7 +229,7 @@ _MAX_UPLOAD_BYTES = 50 * 1024 * 1024
|
||||
@router.post("/upload")
|
||||
async def upload(
|
||||
request: Request,
|
||||
kind: Literal["law", "regulation", "circular", "caselaw"] = Form(...),
|
||||
kind: Literal["law", "regulation", "circular", "caselaw", "tool"] = Form(...),
|
||||
topic_id: int = Form(...),
|
||||
title: str | None = Form(None),
|
||||
identifier: str | None = Form(None),
|
||||
@@ -329,6 +330,9 @@ class SourceUpdate(BaseModel):
|
||||
source_url: str | None = Field(None, max_length=1000)
|
||||
published_at: str | None = None
|
||||
effective_at: str | None = None
|
||||
description: str | None = Field(None, max_length=500)
|
||||
# Optional. None = leave labels unchanged; [] = clear all; [3,7] = exact set.
|
||||
label_ids: list[int] | None = None
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
@@ -336,13 +340,14 @@ async def list_admin_sources(
|
||||
request: Request,
|
||||
topic_id: int | None = None,
|
||||
kind: str | None = None,
|
||||
label_id: int | None = None,
|
||||
limit: int = 200,
|
||||
):
|
||||
_verify_admin(request)
|
||||
if kind and kind not in ("law", "regulation", "circular", "caselaw"):
|
||||
if kind and kind not in ("law", "regulation", "circular", "caselaw", "tool"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid kind: {kind}")
|
||||
items = await kb_admin_sources.list_sources(
|
||||
topic_id=topic_id, kind=kind, limit=limit
|
||||
topic_id=topic_id, kind=kind, label_id=label_id, limit=limit,
|
||||
)
|
||||
return {"items": items, "count": len(items)}
|
||||
|
||||
@@ -350,9 +355,12 @@ async def list_admin_sources(
|
||||
@router.put("/sources/{source_id}")
|
||||
async def update_admin_source(request: Request, source_id: int, body: SourceUpdate):
|
||||
_verify_admin(request)
|
||||
fields = body.model_dump(exclude_unset=True)
|
||||
raw = body.model_dump(exclude_unset=True)
|
||||
label_ids = raw.pop("label_ids", None)
|
||||
try:
|
||||
updated = await kb_admin_sources.update_source(source_id, fields)
|
||||
updated = await kb_admin_sources.update_source(
|
||||
source_id, raw, label_ids=label_ids,
|
||||
)
|
||||
except LookupError:
|
||||
raise HTTPException(status_code=404, detail="Source not found")
|
||||
except ValueError as e:
|
||||
@@ -449,3 +457,267 @@ async def delete_admin_topic(request: Request, topic_id: int):
|
||||
raise HTTPException(status_code=404, detail="Topic not found")
|
||||
except kb_topics.ConflictError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
# ── Phase 6 (Task #20): bulk upload + AI classification + labels ───────────
|
||||
|
||||
@router.post("/upload-batch")
|
||||
async def upload_batch(
|
||||
request: Request,
|
||||
kind: Literal["law", "regulation", "circular", "caselaw", "tool"] = Form(...),
|
||||
topic_id: int = Form(...),
|
||||
# FastAPI accepts a list when the form field name is repeated; the
|
||||
# browser sends `files=...&files=...` for a multi-file <input multiple>
|
||||
# or for FormData.append('files', f) called N times.
|
||||
files: list[UploadFile] = File(...),
|
||||
):
|
||||
"""Multi-file upload. Each file gets its own kb_ingest_job and a shared
|
||||
batch_id (uuid). After insert, asyncio.create_task fires the
|
||||
classify stage in the background. The browser opens the review screen
|
||||
on /admin/kb/batch/{batch_id} and polls until all jobs are
|
||||
'awaiting_review' or 'failed'.
|
||||
"""
|
||||
_verify_admin(request)
|
||||
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="No files in batch")
|
||||
if len(files) > 50:
|
||||
raise HTTPException(status_code=400, detail="Max 50 files per batch")
|
||||
|
||||
try:
|
||||
topic = await kb_topics.get_topic(topic_id)
|
||||
except Exception:
|
||||
topic = None
|
||||
if not topic or not topic.get("is_active"):
|
||||
raise HTTPException(status_code=400, detail=f"Unknown or inactive topic_id={topic_id}")
|
||||
|
||||
requested_by = (
|
||||
request.headers.get("X-User-Name")
|
||||
or request.headers.get("X-Forwarded-User")
|
||||
or None
|
||||
)
|
||||
|
||||
import uuid as _uuid
|
||||
batch_id = _uuid.uuid4().hex
|
||||
|
||||
out_jobs: list[dict] = []
|
||||
for f in files:
|
||||
data = await f.read()
|
||||
if not data:
|
||||
out_jobs.append({"original_filename": f.filename or "upload",
|
||||
"error": "empty file"})
|
||||
continue
|
||||
if len(data) > _MAX_UPLOAD_BYTES:
|
||||
out_jobs.append({"original_filename": f.filename or "upload",
|
||||
"error": f"too large ({len(data)} bytes)"})
|
||||
continue
|
||||
try:
|
||||
job_id = await kb_jobs.create_job(
|
||||
topic_id=topic_id,
|
||||
topic_slug=topic["slug"],
|
||||
kind=kind,
|
||||
original_filename=f.filename or "upload",
|
||||
file_bytes=data,
|
||||
# Empty user metadata at this stage — AI will populate
|
||||
# ai_suggestions, then user confirms via /commit.
|
||||
metadata={},
|
||||
requested_by_user=requested_by,
|
||||
batch_id=batch_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("[upload-batch] create_job failed")
|
||||
out_jobs.append({"original_filename": f.filename or "upload",
|
||||
"error": f"queue failed: {e}"})
|
||||
continue
|
||||
# Fire-and-forget classify stage. The classifier is fail-soft so
|
||||
# the user always reaches the review screen.
|
||||
asyncio.create_task(kb_jobs.process_classify_stage(job_id))
|
||||
out_jobs.append({
|
||||
"job_id": job_id,
|
||||
"original_filename": f.filename or "upload",
|
||||
"status": "queued",
|
||||
})
|
||||
|
||||
return {"batch_id": batch_id, "jobs": out_jobs}
|
||||
|
||||
|
||||
@router.get("/batch/{batch_id}")
|
||||
async def get_batch(request: Request, batch_id: str):
|
||||
_verify_admin(request)
|
||||
items = await kb_jobs.list_jobs_by_batch(batch_id)
|
||||
return {"batch_id": batch_id, "items": items, "count": len(items)}
|
||||
|
||||
|
||||
class JobCommit(BaseModel):
|
||||
"""User-confirmed metadata from the review screen."""
|
||||
kind: Literal["law", "regulation", "circular", "caselaw", "tool"]
|
||||
title: str = Field(..., min_length=1, max_length=500)
|
||||
identifier: str | None = Field(None, max_length=200)
|
||||
source_url: str | None = Field(None, max_length=1000)
|
||||
published_at: str | None = None
|
||||
effective_at: str | None = None
|
||||
summary: str | None = Field(None, max_length=500)
|
||||
# Existing labels — by slug. Lookup is exact match on kb_label.slug.
|
||||
label_slugs: list[str] = Field(default_factory=list, max_length=10)
|
||||
# New labels — display names that the user accepted as "create on commit".
|
||||
# The route slugifies them and creates rows.
|
||||
new_labels: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/commit")
|
||||
async def commit_job(request: Request, job_id: int, body: JobCommit):
|
||||
_verify_admin(request)
|
||||
requested_by = (
|
||||
request.headers.get("X-User-Name")
|
||||
or request.headers.get("X-Forwarded-User")
|
||||
or None
|
||||
)
|
||||
|
||||
# Resolve label_specs. Existing labels: pass slug + name (looked up
|
||||
# by slug, name only used if INSERT happens via the lookup_or_create
|
||||
# helper). New labels: slugify display names; reject empties.
|
||||
label_specs: list[dict] = []
|
||||
seen_slugs: set[str] = set()
|
||||
|
||||
# Existing slugs: look up names from DB so the helper can re-insert
|
||||
# them safely (idempotent on conflict).
|
||||
if body.label_slugs:
|
||||
for slug in body.label_slugs:
|
||||
slug = (slug or "").strip()
|
||||
if not slug or slug in seen_slugs:
|
||||
continue
|
||||
existing = await kb_labels.list_labels(query=slug, limit=1)
|
||||
existing = [l for l in existing if l["slug"] == slug]
|
||||
if existing:
|
||||
label_specs.append({
|
||||
"slug": slug,
|
||||
"name": existing[0]["name"],
|
||||
"topic_id": existing[0].get("topic_id"),
|
||||
})
|
||||
seen_slugs.add(slug)
|
||||
|
||||
# New labels: derive slug from name, dedupe.
|
||||
if body.new_labels:
|
||||
# Find topic for the job so new labels are scoped to it.
|
||||
async with (await get_pool()).acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT topic_id FROM kb_ingest_job WHERE id = $1", job_id,
|
||||
)
|
||||
topic_id = (row or {}).get("topic_id") if row else None
|
||||
for name in body.new_labels:
|
||||
name = (name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
try:
|
||||
slug = kb_labels.slugify(name)
|
||||
except ValueError as e:
|
||||
logger.warning("[commit] bad new label name=%r: %s", name, e)
|
||||
continue
|
||||
if slug in seen_slugs:
|
||||
continue
|
||||
label_specs.append({"slug": slug, "name": name, "topic_id": topic_id})
|
||||
seen_slugs.add(slug)
|
||||
|
||||
fields = {
|
||||
"kind": body.kind,
|
||||
"title": body.title.strip(),
|
||||
"identifier": (body.identifier or "").strip() or None,
|
||||
"source_url": (body.source_url or "").strip() or None,
|
||||
"published_at": body.published_at or None,
|
||||
"effective_at": body.effective_at or None,
|
||||
"summary": (body.summary or "").strip() or None,
|
||||
}
|
||||
|
||||
try:
|
||||
result = await kb_jobs.commit_job(
|
||||
job_id=job_id,
|
||||
fields=fields,
|
||||
label_specs=label_specs,
|
||||
requested_by=requested_by,
|
||||
)
|
||||
except LookupError:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/discard")
|
||||
async def discard_job(request: Request, job_id: int):
|
||||
_verify_admin(request)
|
||||
requested_by = (
|
||||
request.headers.get("X-User-Name")
|
||||
or request.headers.get("X-Forwarded-User")
|
||||
or None
|
||||
)
|
||||
try:
|
||||
return await kb_jobs.discard_job(job_id, requested_by)
|
||||
except LookupError:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
# ── Labels CRUD ────────────────────────────────────────────────────────────
|
||||
|
||||
class LabelCreate(BaseModel):
|
||||
slug: str = Field(..., min_length=1, max_length=200)
|
||||
name: str = Field(..., min_length=1, max_length=300)
|
||||
topic_id: int | None = None
|
||||
|
||||
|
||||
class LabelMerge(BaseModel):
|
||||
into_label_id: int
|
||||
|
||||
|
||||
@router.get("/labels")
|
||||
async def list_admin_labels(
|
||||
request: Request,
|
||||
topic_id: int | None = None,
|
||||
q: str | None = None,
|
||||
limit: int = 50,
|
||||
):
|
||||
_verify_admin(request)
|
||||
items = await kb_labels.list_labels(topic_id=topic_id, query=q, limit=limit)
|
||||
return {"items": items, "count": len(items)}
|
||||
|
||||
|
||||
@router.post("/labels")
|
||||
async def create_admin_label(request: Request, body: LabelCreate):
|
||||
_verify_admin(request)
|
||||
requested_by = (
|
||||
request.headers.get("X-User-Name")
|
||||
or request.headers.get("X-Forwarded-User")
|
||||
or None
|
||||
)
|
||||
try:
|
||||
return await kb_labels.create_label(
|
||||
slug=body.slug,
|
||||
name=body.name,
|
||||
topic_id=body.topic_id,
|
||||
created_by=requested_by,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/labels/{label_id}/merge")
|
||||
async def merge_admin_labels(request: Request, label_id: int, body: LabelMerge):
|
||||
_verify_admin(request)
|
||||
try:
|
||||
return await kb_labels.merge_labels(label_id, body.into_label_id)
|
||||
except LookupError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@router.delete("/labels/{label_id}")
|
||||
async def delete_admin_label(request: Request, label_id: int):
|
||||
_verify_admin(request)
|
||||
try:
|
||||
return await kb_labels.delete_label(label_id)
|
||||
except LookupError:
|
||||
raise HTTPException(status_code=404, detail="Label not found")
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
Reference in New Issue
Block a user