feat(kb): async upload endpoints + ingest job tracking
Phase 2 of the multi-topic KB refactor. Adds three admin endpoints that
back the new "ניהול" tab in the KnowledgeBase EspoCRM extension:
POST /admin/kb/upload — multipart, queues a kb_ingest_job and
fires asyncio.create_task to process
parse → chunk → embed in the
background. Returns {job_id, status}
immediately so the browser doesn't
block on slow PDFs.
GET /admin/kb/jobs — recent jobs, optionally filtered by
topic_id / status / limit.
GET /admin/kb/jobs/{id} — single-job detail for the polling UI.
Migration 002 adds kb_ingest_job (queued/processing/done/failed) with
indexes on (status, created_at) and (topic_id, created_at).
ingest_source now accepts topic_id (NULL → defaults to 1 for back-compat
with the existing scan-inbox cron path) and writes it to kb_source.
S3 layout for new uploads is topic-aware: inbox/<topic_slug>/<kind>/...
The legacy /scan-inbox path on inbox/<kind>/ is unchanged.
Refs Task Master #13 (espocrm-extensions/KnowledgeBase)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
-- Migration 002: track async ingest jobs so the KB UI can show upload status.
|
||||
--
|
||||
-- Phase 2 of Task Master #13 (espocrm-extensions/KnowledgeBase). Today the
|
||||
-- only path into the KB is dropping a file in MinIO and waiting for the
|
||||
-- /admin/kb/scan-inbox cron. Phase 2 adds an upload form in the EspoCRM UI
|
||||
-- that POSTs the file to shira-hermes; we need a row to track the async
|
||||
-- processing so the browser can poll for "queued → processing → done".
|
||||
--
|
||||
-- Idempotent — safe to run twice. Wrapped in a single transaction.
|
||||
--
|
||||
-- Run as postgres superuser:
|
||||
-- docker exec -i <pg-container> psql -U postgres -d insurance_kb \
|
||||
-- -f /tmp/002_ingest_jobs.sql
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kb_ingest_job (
|
||||
id SERIAL PRIMARY KEY,
|
||||
-- Set after success; null while queued/processing and on failure.
|
||||
source_id INTEGER REFERENCES kb_source(id) ON DELETE SET NULL,
|
||||
original_filename TEXT NOT NULL,
|
||||
s3_key TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('law','regulation','circular')),
|
||||
topic_id INTEGER NOT NULL REFERENCES kb_topic(id),
|
||||
-- User-supplied metadata at upload time (title/identifier/dates/source_url).
|
||||
-- Stored as JSONB so the schema doesn't have to grow when we add fields.
|
||||
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued','processing','done','failed')),
|
||||
error_message TEXT,
|
||||
chunks_created INTEGER,
|
||||
-- Auth identity from EspoCRM proxy header. Free-form so renames don't
|
||||
-- break old jobs; UI shows it as informational only.
|
||||
requested_by_user TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Index for the "recent jobs" listing — newest first, optionally filtered
|
||||
-- by status. (status, created_at DESC) covers `WHERE status = $1 ORDER BY ...`
|
||||
-- and the unfiltered `ORDER BY created_at DESC` (PG can scan it backwards).
|
||||
CREATE INDEX IF NOT EXISTS kb_ingest_job_status_created_idx
|
||||
ON kb_ingest_job (status, created_at DESC);
|
||||
|
||||
-- Topic scope is the most common second filter from the UI.
|
||||
CREATE INDEX IF NOT EXISTS kb_ingest_job_topic_idx
|
||||
ON kb_ingest_job (topic_id, created_at DESC);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON kb_ingest_job TO shira_kb;
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCE kb_ingest_job_id_seq TO shira_kb;
|
||||
|
||||
COMMIT;
|
||||
Reference in New Issue
Block a user