Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e7b21919c2 |
@@ -2,5 +2,5 @@
|
|||||||
"currentTag": "master",
|
"currentTag": "master",
|
||||||
"lastSwitched": "2026-04-24T15:47:33.139Z",
|
"lastSwitched": "2026-04-24T15:47:33.139Z",
|
||||||
"branchTagMapping": {},
|
"branchTagMapping": {},
|
||||||
"migrationNoticeShown": false
|
"migrationNoticeShown": true
|
||||||
}
|
}
|
||||||
@@ -150,7 +150,7 @@
|
|||||||
"id": "13",
|
"id": "13",
|
||||||
"title": "Phase 2 — Document upload from the KB UI (file picker + async ingestion + job tracking)",
|
"title": "Phase 2 — Document upload from the KB UI (file picker + async ingestion + job tracking)",
|
||||||
"description": "Today documents enter the KB by manually putting them in the MinIO inbox/ folder and waiting for the n8n cron to call /admin/kb/scan-inbox. End users have no path to upload via the UI. Add a file-upload form in the management panel: user picks a PDF/DOCX/TXT, picks a kind (law/regulation/circular), confirms the topic, optionally fills metadata (title, identifier, dates, source_url), and clicks upload. The file lands in MinIO under inbox/<topic_slug>/<kind>/, a kb_ingest_job row is created, and a background asyncio task processes it (parse → chunk → embed → upsert kb_source). The UI polls the job status and shows progress live; on done it links to the new source in the management table.",
|
"description": "Today documents enter the KB by manually putting them in the MinIO inbox/ folder and waiting for the n8n cron to call /admin/kb/scan-inbox. End users have no path to upload via the UI. Add a file-upload form in the management panel: user picks a PDF/DOCX/TXT, picks a kind (law/regulation/circular), confirms the topic, optionally fills metadata (title, identifier, dates, source_url), and clicks upload. The file lands in MinIO under inbox/<topic_slug>/<kind>/, a kb_ingest_job row is created, and a background asyncio task processes it (parse → chunk → embed → upsert kb_source). The UI polls the job status and shows progress live; on done it links to the new source in the management table.",
|
||||||
"status": "in-progress",
|
"status": "done",
|
||||||
"priority": "high",
|
"priority": "high",
|
||||||
"details": "DB:\n- New table kb_ingest_job: id PK, source_id INTEGER NULL FK kb_source(id) ON DELETE SET NULL, original_filename TEXT, s3_key TEXT, kind TEXT, topic_id INTEGER NOT NULL, metadata_json JSONB, status TEXT NOT NULL CHECK (status IN ('queued','processing','done','failed')), error_message TEXT, chunks_created INTEGER, created_at TIMESTAMPTZ DEFAULT now(), started_at TIMESTAMPTZ, completed_at TIMESTAMPTZ, requested_by_user TEXT.\n- Index: (status, created_at DESC) for the queued list.\n\nshira-hermes endpoints:\n- POST /admin/kb/upload (multipart): receives file, kind, topic_id, optional title/identifier/published_at/effective_at/source_url. Validates topic exists. Writes file to s3 inbox/<topic_slug>/<kind>/<uuid-prefixed filename>. Inserts kb_ingest_job with status=queued. Schedules asyncio.create_task(_process_ingest_job(job_id)) so processing starts immediately, not on the next n8n cron tick. Returns {job_id, status:'queued'}.\n- GET /admin/kb/jobs?topic_id=&status=&limit=: list jobs, newest first.\n- GET /admin/kb/jobs/{id}: single job detail.\n- _process_ingest_job(job_id): UPDATE status='processing', started_at=now. Calls existing kb_ingest.ingest_source. On success: UPDATE status='done', source_id=<new>, chunks_created=<n>, completed_at=now. On failure: UPDATE status='failed', error_message=<exception text>. The upload doesn't block the HTTP request — the user gets the job_id immediately and polls.\n- Reuses scan-inbox advisory lock pattern so a manually-uploaded file plus an n8n cron run don't double-process.\n\nKnowledgeBase extension:\n- EspoCRM PHP proxy for multipart upload — Controllers can read $request->getParsedBody() but multipart needs special handling. Look at how other extensions do it (LegalAssistance/DigitalSignature might have examples).\n- New tab 'ניהול' (visible only to non-portal users; can also gate to admin role later). The tab opens a panel with: (a) Upload form, (b) Sources table for current topic [implemented in Phase 3], (c) Recent jobs.\n- Upload form fields: file picker, kind dropdown, topic dropdown (defaulting to currently-selected topic), optional title/identifier/dates/source_url collapse. Submit → POST KnowledgeBase/action/upload → returns job_id → switch to job-status view that polls every 2s.\n- Job status view: shows the job's status badge (queued/processing/done/failed), elapsed time, and on done: link to the source in the management table (Phase 3).\n\nFile size: enforce 50MB limit at the FastAPI route + server-side. PDFs of עשרות-שעות could approach this.",
|
"details": "DB:\n- New table kb_ingest_job: id PK, source_id INTEGER NULL FK kb_source(id) ON DELETE SET NULL, original_filename TEXT, s3_key TEXT, kind TEXT, topic_id INTEGER NOT NULL, metadata_json JSONB, status TEXT NOT NULL CHECK (status IN ('queued','processing','done','failed')), error_message TEXT, chunks_created INTEGER, created_at TIMESTAMPTZ DEFAULT now(), started_at TIMESTAMPTZ, completed_at TIMESTAMPTZ, requested_by_user TEXT.\n- Index: (status, created_at DESC) for the queued list.\n\nshira-hermes endpoints:\n- POST /admin/kb/upload (multipart): receives file, kind, topic_id, optional title/identifier/published_at/effective_at/source_url. Validates topic exists. Writes file to s3 inbox/<topic_slug>/<kind>/<uuid-prefixed filename>. Inserts kb_ingest_job with status=queued. Schedules asyncio.create_task(_process_ingest_job(job_id)) so processing starts immediately, not on the next n8n cron tick. Returns {job_id, status:'queued'}.\n- GET /admin/kb/jobs?topic_id=&status=&limit=: list jobs, newest first.\n- GET /admin/kb/jobs/{id}: single job detail.\n- _process_ingest_job(job_id): UPDATE status='processing', started_at=now. Calls existing kb_ingest.ingest_source. On success: UPDATE status='done', source_id=<new>, chunks_created=<n>, completed_at=now. On failure: UPDATE status='failed', error_message=<exception text>. The upload doesn't block the HTTP request — the user gets the job_id immediately and polls.\n- Reuses scan-inbox advisory lock pattern so a manually-uploaded file plus an n8n cron run don't double-process.\n\nKnowledgeBase extension:\n- EspoCRM PHP proxy for multipart upload — Controllers can read $request->getParsedBody() but multipart needs special handling. Look at how other extensions do it (LegalAssistance/DigitalSignature might have examples).\n- New tab 'ניהול' (visible only to non-portal users; can also gate to admin role later). The tab opens a panel with: (a) Upload form, (b) Sources table for current topic [implemented in Phase 3], (c) Recent jobs.\n- Upload form fields: file picker, kind dropdown, topic dropdown (defaulting to currently-selected topic), optional title/identifier/dates/source_url collapse. Submit → POST KnowledgeBase/action/upload → returns job_id → switch to job-status view that polls every 2s.\n- Job status view: shows the job's status badge (queued/processing/done/failed), elapsed time, and on done: link to the source in the management table (Phase 3).\n\nFile size: enforce 50MB limit at the FastAPI route + server-side. PDFs of עשרות-שעות could approach this.",
|
||||||
"testStrategy": "1) Upload a small test PDF (~1MB). Job goes queued → processing → done in <30s. UI shows progress live. 2) Upload a corrupt file. Job ends in failed with a useful error_message. 3) Upload while another large ingest is running — job shows 'processing' but doesn't deadlock. 4) Refresh the page mid-upload — job status restored from sessionStorage + a fresh poll. 5) /admin/kb/jobs returns the upload's row.",
|
"testStrategy": "1) Upload a small test PDF (~1MB). Job goes queued → processing → done in <30s. UI shows progress live. 2) Upload a corrupt file. Job ends in failed with a useful error_message. 3) Upload while another large ingest is running — job shows 'processing' but doesn't deadlock. 4) Refresh the page mid-upload — job status restored from sessionStorage + a fresh poll. 5) /admin/kb/jobs returns the upload's row.",
|
||||||
@@ -159,7 +159,7 @@
|
|||||||
"12"
|
"12"
|
||||||
],
|
],
|
||||||
"createdAt": "2026-04-25T13:30:00Z",
|
"createdAt": "2026-04-25T13:30:00Z",
|
||||||
"updatedAt": "2026-04-25T16:20:11.931Z"
|
"updatedAt": "2026-04-25T16:47:46.599Z"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "14",
|
"id": "14",
|
||||||
@@ -215,13 +215,37 @@
|
|||||||
"15"
|
"15"
|
||||||
],
|
],
|
||||||
"createdAt": "2026-04-25T13:30:00Z"
|
"createdAt": "2026-04-25T13:30:00Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "18",
|
||||||
|
"title": "feat(kb): add 'caselaw' (פסיקה) as a fourth kind",
|
||||||
|
"description": "Add 'caselaw' alongside law/regulation/circular across the whole stack. Touches: DB CHECK constraints (migration 003), shira-hermes Python (Literal types, _KINDS, kind validation, chunker section detection), EspoCRM Controller/Service validators, template dropdowns, JS kindHe mapping.",
|
||||||
|
"details": "DB: migration 003_caselaw_kind.sql alters CHECK on kb_source.kind and kb_ingest_job.kind to include 'caselaw'. shira-hermes: update Literal types in ingest.py, admin_kb.py, kb_public.py SearchRequest; add 'caselaw' to s3.py _KINDS; chunker.py — fall back to generic section detector (regex for 'פסק דין', 'תיק' / case number, paragraph numbers). EspoCRM: update validators in Controller postActionUpload + Service search/uploadFile; add 'caselaw' option to kind <select> in index.tpl (search box + upload form); update kindHe in index.js.",
|
||||||
|
"testStrategy": "",
|
||||||
|
"status": "in-progress",
|
||||||
|
"dependencies": [],
|
||||||
|
"priority": "medium",
|
||||||
|
"subtasks": [],
|
||||||
|
"updatedAt": "2026-04-25T16:49:30.816Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "19",
|
||||||
|
"title": "feat(kb): tag/label sub-topics within a topic (e.g. 'ילד נכה')",
|
||||||
|
"description": "Group multiple sources within a topic by sub-subject so users can find e.g. all 'ילד נכה' circulars together. Recommendation: many-to-many kb_label + kb_source_label so a source can carry multiple tags reflecting that legal docs commonly touch several subjects.",
|
||||||
|
"status": "pending",
|
||||||
|
"priority": "medium",
|
||||||
|
"details": "Three approaches considered:\n\n1. Free-text 'subject' column on kb_source (~1d): cheapest, typo-prone (ילד נכה / ילדים נכים → separate buckets).\n\n2. Many-to-many labels (~3d, RECOMMENDED): kb_label(id, topic_id, slug UNIQUE per topic, name) + kb_source_label(source_id, label_id, PK both). UI shows label chips on sources; click to filter. Naturally handles cross-cutting subjects (a circular tagged both 'ילד נכה' AND 'אזרח ותיק'). Needs admin UI to manage labels.\n\n3. Hierarchical sub-topics under kb_topic (~5d): clean tree, but a source can only live in one branch — wrong fit for the legal world. Also clashes with planned Phase 4 topic-CRUD UI.\n\nPragmatic path: ship #1 first (subject TEXT field on kb_source + dropdown of seen values + browse-view grouping); upgrade to #2 once we see how labels are used in practice.",
|
||||||
|
"testStrategy": "Upload 3 circulars all in 'ילד נכה' subject; verify they appear grouped together in browse view; verify search results show the subject chip; verify topic switch resets the subject filter.",
|
||||||
|
"subtasks": [],
|
||||||
|
"dependencies": [],
|
||||||
|
"createdAt": "2026-04-25T16:54:54.478470Z"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"lastModified": "2026-04-25T16:20:11.932Z",
|
"lastModified": "2026-04-25T16:49:30.816Z",
|
||||||
"taskCount": 17,
|
"taskCount": 18,
|
||||||
"completedCount": 9,
|
"completedCount": 10,
|
||||||
"tags": [
|
"tags": [
|
||||||
"master"
|
"master"
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -36,6 +36,7 @@
|
|||||||
<option value="law">חוק</option>
|
<option value="law">חוק</option>
|
||||||
<option value="regulation">תקנות</option>
|
<option value="regulation">תקנות</option>
|
||||||
<option value="circular">חוזרים</option>
|
<option value="circular">חוזרים</option>
|
||||||
|
<option value="caselaw">פסיקה</option>
|
||||||
</select>
|
</select>
|
||||||
<button type="button" class="btn btn-primary" data-action="submit">חפש</button>
|
<button type="button" class="btn btn-primary" data-action="submit">חפש</button>
|
||||||
<button type="button" class="btn btn-default" data-action="clearResults"
|
<button type="button" class="btn btn-default" data-action="clearResults"
|
||||||
@@ -88,6 +89,7 @@
|
|||||||
<option value="law">חוק</option>
|
<option value="law">חוק</option>
|
||||||
<option value="regulation">תקנות</option>
|
<option value="regulation">תקנות</option>
|
||||||
<option value="circular">חוזר</option>
|
<option value="circular">חוזר</option>
|
||||||
|
<option value="caselaw">פסיקה</option>
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;">
|
<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;">
|
||||||
|
|||||||
@@ -247,7 +247,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) {
|
|||||||
done: '<span class="label label-success">הושלם</span>',
|
done: '<span class="label label-success">הושלם</span>',
|
||||||
failed: '<span class="label label-danger">נכשל</span>',
|
failed: '<span class="label label-danger">נכשל</span>',
|
||||||
};
|
};
|
||||||
const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר'};
|
const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר', caselaw: 'פסיקה'};
|
||||||
const rows = items.map(j => {
|
const rows = items.map(j => {
|
||||||
const when = j.completed_at || j.started_at || j.created_at || '';
|
const when = j.completed_at || j.started_at || j.created_at || '';
|
||||||
const whenShort = String(when).slice(0, 19).replace('T', ' ');
|
const whenShort = String(when).slice(0, 19).replace('T', ' ');
|
||||||
@@ -813,7 +813,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const initialIdx = (typeof selectedIdx === 'number' && hits[selectedIdx]) ? selectedIdx : 0;
|
const initialIdx = (typeof selectedIdx === 'number' && hits[selectedIdx]) ? selectedIdx : 0;
|
||||||
const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר'};
|
const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר', caselaw: 'פסיקה'};
|
||||||
const self = this;
|
const self = this;
|
||||||
|
|
||||||
const isPdfBacked = h => h && (h.kind === 'regulation' || h.kind === 'circular');
|
const isPdfBacked = h => h && (h.kind === 'regulation' || h.kind === 'circular');
|
||||||
@@ -1112,7 +1112,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) {
|
|||||||
answerHtml = '<div style="white-space:pre-wrap;">' + this.escape(text || '') + '</div>';
|
answerHtml = '<div style="white-space:pre-wrap;">' + this.escape(text || '') + '</div>';
|
||||||
}
|
}
|
||||||
|
|
||||||
const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר'};
|
const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר', caselaw: 'פסיקה'};
|
||||||
|
|
||||||
// Dedup sources by source_id (first occurrence = most relevant),
|
// Dedup sources by source_id (first occurrence = most relevant),
|
||||||
// but remember every page that came up per source so users can
|
// but remember every page that came up per source so users can
|
||||||
@@ -1252,8 +1252,8 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) {
|
|||||||
$list.html('<div class="text-muted">אין מקורות.</div>');
|
$list.html('<div class="text-muted">אין מקורות.</div>');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזרים'};
|
const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזרים', caselaw: 'פסיקה'};
|
||||||
const grouped = {law: [], regulation: [], circular: []};
|
const grouped = {law: [], regulation: [], circular: [], caselaw: []};
|
||||||
this._sources.forEach(s => {
|
this._sources.forEach(s => {
|
||||||
if (grouped[s.kind]) grouped[s.kind].push(s);
|
if (grouped[s.kind]) grouped[s.kind].push(s);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -145,8 +145,8 @@ class KnowledgeBase
|
|||||||
// For multipart/form-data EspoCRM's getParsedBody() returns an empty
|
// For multipart/form-data EspoCRM's getParsedBody() returns an empty
|
||||||
// stdClass; the form fields land in $_POST as PHP parses them.
|
// stdClass; the form fields land in $_POST as PHP parses them.
|
||||||
$kind = $_POST['kind'] ?? null;
|
$kind = $_POST['kind'] ?? null;
|
||||||
if (!in_array($kind, ['law', 'regulation', 'circular'], true)) {
|
if (!in_array($kind, ['law', 'regulation', 'circular', 'caselaw'], true)) {
|
||||||
throw new BadRequest('kind must be one of: law, regulation, circular.');
|
throw new BadRequest('kind must be one of: law, regulation, circular, caselaw.');
|
||||||
}
|
}
|
||||||
$topicId = $this->coerceTopicId($_POST['topicId'] ?? $_POST['topic_id'] ?? null);
|
$topicId = $this->coerceTopicId($_POST['topicId'] ?? $_POST['topic_id'] ?? null);
|
||||||
if ($topicId === null) {
|
if ($topicId === null) {
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ class KnowledgeBaseService
|
|||||||
public function search(string $query, string $kind, int $topK, ?int $topicId = null): array
|
public function search(string $query, string $kind, int $topK, ?int $topicId = null): array
|
||||||
{
|
{
|
||||||
$topK = max(1, min(15, $topK));
|
$topK = max(1, min(15, $topK));
|
||||||
if (!in_array($kind, ['any', 'law', 'regulation', 'circular'], true)) {
|
if (!in_array($kind, ['any', 'law', 'regulation', 'circular', 'caselaw'], true)) {
|
||||||
$kind = 'any';
|
$kind = 'any';
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,8 +88,8 @@ class KnowledgeBaseService
|
|||||||
if (!is_readable($tmpPath)) {
|
if (!is_readable($tmpPath)) {
|
||||||
throw new Error('Uploaded file is not readable on disk.');
|
throw new Error('Uploaded file is not readable on disk.');
|
||||||
}
|
}
|
||||||
if (!in_array($kind, ['law', 'regulation', 'circular'], true)) {
|
if (!in_array($kind, ['law', 'regulation', 'circular', 'caselaw'], true)) {
|
||||||
throw new BadRequest('kind must be one of: law, regulation, circular.');
|
throw new BadRequest('kind must be one of: law, regulation, circular, caselaw.');
|
||||||
}
|
}
|
||||||
|
|
||||||
$url = $this->getBaseUrl() . '/admin/kb/upload';
|
$url = $this->getBaseUrl() . '/admin/kb/upload';
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "KnowledgeBase",
|
"name": "KnowledgeBase",
|
||||||
"module": "KnowledgeBase",
|
"module": "KnowledgeBase",
|
||||||
"version": "0.4.0",
|
"version": "0.5.0",
|
||||||
"acceptableVersions": [
|
"acceptableVersions": [
|
||||||
">=8.0.0"
|
">=8.0.0"
|
||||||
],
|
],
|
||||||
|
|||||||
Reference in New Issue
Block a user