diff --git a/.taskmaster/tasks/tasks.json b/.taskmaster/tasks/tasks.json index acfba35..c5f7fc6 100644 --- a/.taskmaster/tasks/tasks.json +++ b/.taskmaster/tasks/tasks.json @@ -165,7 +165,7 @@ "id": "14", "title": "Phase 3 — Management panel UI (sources table, edit metadata, delete, re-ingest)", "description": "Today the only way to manage existing sources is via SQL on the shared PG (DELETE FROM kb_source WHERE id=…) plus an mc command on MinIO. End users need a panel where they can: see all sources for the selected topic, with chunk counts and ingestion dates; edit source metadata (title, identifier, dates, source_url); delete a source (with confirm and clear cascade messaging); re-ingest a source (re-fetch its file from MinIO processed/, drop chunks, re-run parse+chunk+embed); see the most recent ingest jobs (success + failure both). The panel sits in the same KB extension under a new tab 'ניהול'.", - "status": "pending", + "status": "in-progress", "priority": "high", "details": "shira-hermes endpoints:\n- GET /admin/kb/sources?topic_id=&kind=&limit=: list with id, kind, title, identifier, chunk_count, ingestion_date, last_ingest_status (joining kb_ingest_job).\n- PUT /admin/kb/sources/{id}: update editable fields (title, identifier, source_url, published_at, effective_at).\n- DELETE /admin/kb/sources/{id}: hard delete (cascades chunks via FK, drops processed/ files in S3 too).\n- POST /admin/kb/sources/{id}/reingest: requires source_id; finds the original file at original_path s3:// URI; creates a new kb_ingest_job row with status=queued and the existing source_id; background task replaces chunks+embeddings (does NOT change source_id, so all v0.1.11 chunk_index references remain valid IF the chunker output is stable; otherwise old _lastSearch sessionStorage will point at chunk_indexes that no longer exist — flag for testing). Should optionally just delete + re-create the source — simpler.\n\nEspoCRM client:\n- 'ניהול' tab. Top: header showing topic name + a count badge ('X מסמכים בנושא'). Below: 3 collapsed sections — uploaded jobs (collapsed if all done), sources table, recent failures.\n- Sources table: id, kind label, title, identifier, chunk_count, ingestion_date. Rightmost column: action buttons (✏ edit, 🗑 delete, ↻ re-ingest, 👁 view in browse mode).\n- Edit: opens a modal with the editable fields. PUT on save.\n- Delete: confirm modal that shows chunk_count + 'this will also remove the PDF from the search results'. Two-step confirmation if source has >100 chunks.\n- Re-ingest: confirms; submits POST /reingest; surfaces in the jobs section.\n- Use jQuery + Espo.Ui.dialog patterns consistent with the rest of the extension.\n\nACL gate: this tab visible only to users with role 'KB Admin' (define in EspoCRM if not present) or fall back to !isPortal for now.", "testStrategy": "1) Open 'ניהול' tab. See the 6 existing sources for ביטוח לאומי. 2) Edit one source's title — refresh → new title sticks. 3) Re-ingest ספר הליקויים. Job appears in the jobs section processing → done. Source still searchable end-to-end after. 4) Delete one source. Confirm modal appears with chunk count. After delete: source gone from /kb/sources, /kb/search no longer surfaces it. 5) Failed re-ingest (e.g. delete the file in MinIO first) — surfaces in failures section with error_message.", @@ -173,7 +173,8 @@ "dependencies": [ "13" ], - "createdAt": "2026-04-25T13:30:00Z" + "createdAt": "2026-04-25T13:30:00Z", + "updatedAt": "2026-04-25T17:03:34.486Z" }, { "id": "15", @@ -222,11 +223,11 @@ "description": "Add 'caselaw' alongside law/regulation/circular across the whole stack. Touches: DB CHECK constraints (migration 003), shira-hermes Python (Literal types, _KINDS, kind validation, chunker section detection), EspoCRM Controller/Service validators, template dropdowns, JS kindHe mapping.", "details": "DB: migration 003_caselaw_kind.sql alters CHECK on kb_source.kind and kb_ingest_job.kind to include 'caselaw'. shira-hermes: update Literal types in ingest.py, admin_kb.py, kb_public.py SearchRequest; add 'caselaw' to s3.py _KINDS; chunker.py — fall back to generic section detector (regex for 'פסק דין', 'תיק' / case number, paragraph numbers). EspoCRM: update validators in Controller postActionUpload + Service search/uploadFile; add 'caselaw' option to kind + + + + + + + + + +
+
טוען מקורות…
+
+

העלאת מסמך חדש

טוען מקורות…
'); + Espo.Ajax.getRequest('KnowledgeBase/action/adminSources', { + topicId: topicAtFetch, + limit: 200, + }).then(res => { + if (self.topicId !== topicAtFetch) return; + _adminSourcesByTopic[topicAtFetch] = res && res.items || []; + self._renderAdminSourcesTable(); + }).catch(err => { + console.error('KB: failed to load admin sources', err); + $table.html('
שגיאה בטעינת מקורות.
'); + }); + }, + + _renderAdminSourcesTable: function () { + const $table = this.$el.find('.kb-sources-table'); + if (!$table.length || this.topicId == null) return; + const all = _adminSourcesByTopic[this.topicId] || []; + const filterKind = (this.$el.find('.kb-sources-kind-filter').val() || '').trim(); + const items = filterKind ? all.filter(s => s.kind === filterKind) : all; + + if (!items.length) { + $table.html('
' + (all.length ? 'אין מקורות בסוג זה.' : 'אין מקורות בנושא הזה.') + '
'); + return; + } + + const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר', caselaw: 'פסיקה'}; + const rows = items.map(s => { + const li = s.last_ingest || {}; + const failedFlag = li.status === 'failed' + ? `כשל בקריאה אחרונה` + : ''; + const updated = String(s.updated_at || s.created_at || '').slice(0, 10); + const ident = s.identifier ? this.escape(s.identifier) : '—'; + return ` + ${kindHe[s.kind] || s.kind} + +
${this.escape(s.title || '—')}
+
${ident} · ${this.escape(updated)} ${failedFlag}
+ + ${s.chunk_count} + + + + + + `; + }).join(''); + + $table.html(` + + + + + ${rows} +
סוגכותרת / מזהה / עודכןקטעים
+ `); + }, + + _findSourceById: function (id) { + const arr = _adminSourcesByTopic[this.topicId] || []; + return arr.find(s => s.id === id); + }, + + _openSourceEditForm: function (id) { + const src = this._findSourceById(id); + if (!src) return; + const $row = this.$el.find(`.kb-sources-table tr[data-id="${id}"]`); + if (!$row.length) return; + const safe = (v) => this.escape(v == null ? '' : String(v)); + const dateOnly = (v) => v ? String(v).slice(0, 10) : ''; + $row.html(` + +
+
+ + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+
+ + `); + $row.find('input[data-edit="title"]').trigger('focus'); + }, + + _saveSourceEdit: function (id) { + const self = this; + const $row = this.$el.find(`.kb-sources-table tr[data-id="${id}"]`); + if (!$row.length) return; + const payload = {id}; + $row.find('input[data-edit]').each(function () { + const k = $(this).data('edit'); + payload[k] = ($(this).val() || '').trim(); + }); + const $btn = $row.find('[data-action="saveEdit"]'); + $btn.prop('disabled', true).text('שומר…'); + Espo.Ajax.postRequest('KnowledgeBase/action/updateSource', payload) + .then(updated => { + // Replace cached row in place so the table re-renders with new metadata. + const arr = _adminSourcesByTopic[self.topicId] || []; + const idx = arr.findIndex(s => s.id === id); + if (idx >= 0) { + arr[idx] = Object.assign({}, arr[idx], updated); + } + self._renderAdminSourcesTable(); + }) + .catch(err => { + $btn.prop('disabled', false).text('שמור'); + const msg = (err && err.responseText) || 'שגיאה לא ידועה'; + alert('שמירה נכשלה: ' + msg.slice(0, 200)); + }); + }, + + _deleteSource: function (id) { + const src = this._findSourceById(id); + if (!src) return; + const cn = src.chunk_count || 0; + const titleShort = (src.title || '').slice(0, 60); + let warn = `למחוק את "${titleShort}"?\n\n` + + `מקור עם ${cn} קטעים יימחק לצמיתות מחיפוש ה-KB.\n` + + `הקובץ ב-MinIO גם יימחק.`; + if (!confirm(warn)) return; + if (cn > 100) { + if (!confirm(`למקור הזה יש מעל 100 קטעים (${cn}). אישור סופי?`)) return; + } + const self = this; + Espo.Ajax.postRequest('KnowledgeBase/action/deleteSource', {id}) + .then(res => { + // Drop from cache and re-render. + const arr = _adminSourcesByTopic[self.topicId] || []; + _adminSourcesByTopic[self.topicId] = arr.filter(s => s.id !== id); + self._renderAdminSourcesTable(); + }) + .catch(err => { + const msg = (err && err.responseText) || 'שגיאה לא ידועה'; + alert('מחיקה נכשלה: ' + msg.slice(0, 200)); + }); + }, + + _reingestSource: function (id) { + const src = this._findSourceById(id); + if (!src) return; + const titleShort = (src.title || '').slice(0, 60); + if (!confirm(`לעבד מחדש את "${titleShort}"?\n\nהקטעים הקיימים יימחקו והקובץ יעובר parse + chunk + embed מחדש.\nהמטא-דאטה (כותרת, מזהה, תאריכים) נשמרים.`)) return; + const self = this; + Espo.Ajax.postRequest('KnowledgeBase/action/reingestSource', {id}) + .then(res => { + if (res && res.job_id) { + // Surface in the upload-status banner + jobs list, like an upload would. + self._renderJobStatus({ + original_filename: src.title || ('source #' + id), + status: res.status || 'queued', + }); + self._startJobPolling(res.job_id); + // Refresh sources after the polling reports done. + } + }) + .catch(err => { + const msg = (err && err.responseText) || 'שגיאה לא ידועה'; + alert('עיבוד מחדש נכשל: ' + msg.slice(0, 200)); + }); + }, + // Live status panel rendered above the upload form during processing. _renderJobStatus: function (job) { const $form = this.$el.find('.kb-upload-form'); @@ -316,8 +544,12 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { self._stopJobPolling(); _activeJobId = null; _activeJobTopicId = null; - // Refresh the recent-jobs list to surface the new row. + // Refresh the recent-jobs list to surface the new row, + // and the admin sources table — re-ingest changed + // chunk_count + last_ingest fields, and a fresh + // upload added a new row. self._loadJobsList(); + self._loadAdminSources(/* force */ true); } }) .catch(err => { @@ -475,6 +707,9 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { this._stopJobPolling(); _activeJobId = null; _activeJobTopicId = null; + // Phase 3: a topic change invalidates the admin sources view + // (different scope) — drop the cache so afterRender re-fetches. + _adminSourcesByTopic = {}; this.stopAskProgress(); this.setLoading(false); this.$el.find('.kb-results').empty(); diff --git a/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php b/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php index 0c05420..78174b4 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php +++ b/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php @@ -191,4 +191,65 @@ class KnowledgeBase } return $this->getService()->getJob((int) $jobId); } + + // ── Phase 3: source management (Task #14) ─────────────────────────────── + + public function getActionAdminSources(Request $request, Response $response): array + { + $this->checkAccess(); + $topicId = $this->coerceTopicId($request->getQueryParam('topicId')); + $kind = $request->getQueryParam('kind'); + $limit = $request->getQueryParam('limit'); + $limitInt = ($limit !== null && is_numeric($limit)) ? (int) $limit : 200; + return $this->getService()->listAdminSources($topicId, $kind, $limitInt); + } + + public function postActionUpdateSource(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + if (!$id || !is_numeric($id)) { + throw new BadRequest('id (numeric) is required.'); + } + // Whitelist what we forward — never let arbitrary body fields hit + // upstream's PUT body. Empty strings are explicitly preserved as + // null clearing (the user wiping a date or identifier). + $payload = []; + foreach (['title', 'identifier', 'source_url', 'sourceUrl', 'published_at', 'publishedAt', 'effective_at', 'effectiveAt'] as $k) { + if (property_exists($body, $k)) { + $canon = $k; + if ($k === 'sourceUrl') $canon = 'source_url'; + if ($k === 'publishedAt') $canon = 'published_at'; + if ($k === 'effectiveAt') $canon = 'effective_at'; + $payload[$canon] = $body->$k; + } + } + return $this->getService()->updateAdminSource((int) $id, $payload); + } + + public function postActionDeleteSource(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + if (!$id || !is_numeric($id)) { + throw new BadRequest('id (numeric) is required.'); + } + return $this->getService()->deleteAdminSource((int) $id); + } + + public function postActionReingestSource(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + if (!$id || !is_numeric($id)) { + throw new BadRequest('id (numeric) is required.'); + } + return $this->getService()->reingestAdminSource( + (int) $id, + (string) $this->user->get('userName') + ); + } } diff --git a/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json b/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json index 2a9241f..e5ff6db 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json +++ b/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json @@ -62,5 +62,37 @@ "controller": "KnowledgeBase", "action": "job" } + }, + { + "route": "/KnowledgeBase/action/adminSources", + "method": "get", + "params": { + "controller": "KnowledgeBase", + "action": "adminSources" + } + }, + { + "route": "/KnowledgeBase/action/updateSource", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "updateSource" + } + }, + { + "route": "/KnowledgeBase/action/deleteSource", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "deleteSource" + } + }, + { + "route": "/KnowledgeBase/action/reingestSource", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "reingestSource" + } } ] diff --git a/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php b/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php index 3f27bd6..7d567e0 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php +++ b/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php @@ -173,6 +173,83 @@ class KnowledgeBaseService return $this->get('/admin/kb/jobs/' . $jobId); } + // ── Phase 3: admin source management (Task #14) ───────────────────────── + + public function listAdminSources(?int $topicId, ?string $kind, int $limit): array + { + $limit = max(1, min(500, $limit)); + $qs = ['limit' => $limit]; + if ($topicId !== null) $qs['topic_id'] = $topicId; + if ($kind !== null && $kind !== '') $qs['kind'] = $kind; + return $this->get('/admin/kb/sources?' . http_build_query($qs)); + } + + public function updateAdminSource(int $sourceId, array $fields): array + { + return $this->request('PUT', '/admin/kb/sources/' . $sourceId, $fields); + } + + public function deleteAdminSource(int $sourceId): array + { + return $this->request('DELETE', '/admin/kb/sources/' . $sourceId, null); + } + + public function reingestAdminSource(int $sourceId, string $username): array + { + // Re-ingest needs the user identity for audit; piggyback on the + // X-User-Name header that the upload route already understands. + return $this->postWithUser( + '/admin/kb/sources/' . $sourceId . '/reingest', + null, + $username + ); + } + + private function postWithUser(string $path, ?array $payload, string $username): array + { + $url = $this->getBaseUrl() . $path; + $apiKey = $this->getApiKey(); + + $headers = ['Accept: application/json', 'X-User-Name: ' . $username]; + if ($apiKey) $headers[] = 'X-Api-Key: ' . $apiKey; + + $ch = curl_init($url); + $opts = [ + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_RETURNTRANSFER => true, + CURLOPT_TIMEOUT => 180, + CURLOPT_CONNECTTIMEOUT => 10, + ]; + if ($payload !== null) { + $headers[] = 'Content-Type: application/json'; + $opts[CURLOPT_POSTFIELDS] = json_encode($payload); + } + $opts[CURLOPT_HTTPHEADER] = $headers; + curl_setopt_array($ch, $opts); + + $body = curl_exec($ch); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $error = curl_error($ch); + curl_close($ch); + + if ($error) { + $this->log->error("KnowledgeBase: HTTP error calling {$url}: {$error}"); + throw new Error("Failed to reach Knowledge Base: {$error}"); + } + if ($httpCode < 200 || $httpCode >= 300) { + $detail = $this->decodeDetail($body) ?: "HTTP {$httpCode}"; + if ($httpCode >= 400 && $httpCode < 500) { + throw new BadRequest($detail); + } + throw new Error("Knowledge Base returned HTTP {$httpCode}"); + } + $decoded = json_decode($body, true); + if (!is_array($decoded)) { + throw new Error('Invalid JSON from Knowledge Base.'); + } + return $decoded; + } + private function guessMime(string $filename): string { $lower = strtolower($filename); diff --git a/manifest.json b/manifest.json index bac973c..16c247b 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "name": "KnowledgeBase", "module": "KnowledgeBase", - "version": "0.5.0", + "version": "0.6.0", "acceptableVersions": [ ">=8.0.0" ],