chaim
0cb89fbbe8
Phase 2 of the multi-topic KB refactor. Adds three admin endpoints that
back the new "ניהול" tab in the KnowledgeBase EspoCRM extension:
POST /admin/kb/upload — multipart, queues a kb_ingest_job and
fires asyncio.create_task to process
parse → chunk → embed in the
background. Returns {job_id, status}
immediately so the browser doesn't
block on slow PDFs.
GET /admin/kb/jobs — recent jobs, optionally filtered by
topic_id / status / limit.
GET /admin/kb/jobs/{id} — single-job detail for the polling UI.
Migration 002 adds kb_ingest_job (queued/processing/done/failed) with
indexes on (status, created_at) and (topic_id, created_at).
ingest_source now accepts topic_id (NULL → defaults to 1 for back-compat
with the existing scan-inbox cron path) and writes it to kb_source.
S3 layout for new uploads is topic-aware: inbox/<topic_slug>/<kind>/...
The legacy /scan-inbox path on inbox/<kind>/ is unchanged.
Refs Task Master #13 (espocrm-extensions/KnowledgeBase)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
54 lines
2.4 KiB
PL/PgSQL
54 lines
2.4 KiB
PL/PgSQL
-- Migration 002: track async ingest jobs so the KB UI can show upload status.
|
|
--
|
|
-- Phase 2 of Task Master #13 (espocrm-extensions/KnowledgeBase). Today the
|
|
-- only path into the KB is dropping a file in MinIO and waiting for the
|
|
-- /admin/kb/scan-inbox cron. Phase 2 adds an upload form in the EspoCRM UI
|
|
-- that POSTs the file to shira-hermes; we need a row to track the async
|
|
-- processing so the browser can poll for "queued → processing → done".
|
|
--
|
|
-- Idempotent — safe to run twice. Wrapped in a single transaction.
|
|
--
|
|
-- Run as postgres superuser:
|
|
-- docker exec -i <pg-container> psql -U postgres -d insurance_kb \
|
|
-- -f /tmp/002_ingest_jobs.sql
|
|
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS kb_ingest_job (
|
|
id SERIAL PRIMARY KEY,
|
|
-- Set after success; null while queued/processing and on failure.
|
|
source_id INTEGER REFERENCES kb_source(id) ON DELETE SET NULL,
|
|
original_filename TEXT NOT NULL,
|
|
s3_key TEXT NOT NULL,
|
|
kind TEXT NOT NULL CHECK (kind IN ('law','regulation','circular')),
|
|
topic_id INTEGER NOT NULL REFERENCES kb_topic(id),
|
|
-- User-supplied metadata at upload time (title/identifier/dates/source_url).
|
|
-- Stored as JSONB so the schema doesn't have to grow when we add fields.
|
|
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
status TEXT NOT NULL DEFAULT 'queued'
|
|
CHECK (status IN ('queued','processing','done','failed')),
|
|
error_message TEXT,
|
|
chunks_created INTEGER,
|
|
-- Auth identity from EspoCRM proxy header. Free-form so renames don't
|
|
-- break old jobs; UI shows it as informational only.
|
|
requested_by_user TEXT,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
started_at TIMESTAMPTZ,
|
|
completed_at TIMESTAMPTZ
|
|
);
|
|
|
|
-- Index for the "recent jobs" listing — newest first, optionally filtered
|
|
-- by status. (status, created_at DESC) covers `WHERE status = $1 ORDER BY ...`
|
|
-- and the unfiltered `ORDER BY created_at DESC` (PG can scan it backwards).
|
|
CREATE INDEX IF NOT EXISTS kb_ingest_job_status_created_idx
|
|
ON kb_ingest_job (status, created_at DESC);
|
|
|
|
-- Topic scope is the most common second filter from the UI.
|
|
CREATE INDEX IF NOT EXISTS kb_ingest_job_topic_idx
|
|
ON kb_ingest_job (topic_id, created_at DESC);
|
|
|
|
GRANT SELECT, INSERT, UPDATE, DELETE ON kb_ingest_job TO shira_kb;
|
|
GRANT USAGE, SELECT, UPDATE ON SEQUENCE kb_ingest_job_id_seq TO shira_kb;
|
|
|
|
COMMIT;
|