From 6781ac4e379468ba3f0fbe2a0491c5b0deb6fba2 Mon Sep 17 00:00:00 2001 From: Chaim Date: Sat, 25 Apr 2026 16:46:42 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=202=20=E2=80=94=20document=20uplo?= =?UTF-8?q?ad=20from=20KB=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a "ניהול" tab with multipart file upload, async job tracking, and a recent-jobs list with live status polling. Browser POSTs to /KnowledgeBase/action/upload (PHP proxy), which forwards as multipart to shira-hermes /admin/kb/upload. shira-hermes returns a job_id immediately and processes parse/chunk/embed in a background task; the browser polls every 2s until status hits done|failed. New EspoCRM endpoints: POST /KnowledgeBase/action/upload (multipart, $_FILES['file']) GET /KnowledgeBase/action/jobs (list, filtered by topicId/status) GET /KnowledgeBase/action/job?id (single-job detail) The X-User-Name header is forwarded so kb_ingest_job records who uploaded each file. Cross-topic guard in switchTopic stops polling when the user changes topics mid-upload (the job continues server-side). Depends on shira-hermes commit 0cb89fb (admin upload endpoints + migration 002). Refs Task Master #13 Co-Authored-By: Claude Opus 4.6 (1M context) --- .taskmaster/tasks/tasks.json | 13 +- .../knowledge-base/res/templates/kb/index.tpl | 71 ++++++ .../knowledge-base/src/views/kb/index.js | 223 ++++++++++++++++++ .../Controllers/KnowledgeBase.php | 70 ++++++ .../KnowledgeBase/Resources/routes.json | 24 ++ .../Services/KnowledgeBaseService.php | 130 ++++++++++ manifest.json | 2 +- 7 files changed, 526 insertions(+), 7 deletions(-) diff --git a/.taskmaster/tasks/tasks.json b/.taskmaster/tasks/tasks.json index 09e10ba..05af49e 100644 --- a/.taskmaster/tasks/tasks.json +++ b/.taskmaster/tasks/tasks.json @@ -137,20 +137,20 @@ "id": "12", "title": "Phase 1 — Multi-topic KB foundation (DB schema, topic-aware endpoints, topic selector)", "description": "Generalize the KB from being insurance-only to supporting multiple legal domains within one CRM (e.g. ביטוח לאומי, דיני עבודה, דין פלילי). A 'topic' is the firm-level grouping; each kb_source belongs to exactly one topic. Search/ask are scoped to a single topic at a time, chosen from a topic dropdown the user controls. The system_prompt for /kb/ask becomes per-topic so Shira's domain context is correct (today the prompt hardcodes 'Israeli National Insurance'). Backwards-compatible migration: all 6 existing sources move to topic_id=1 named 'ביטוח לאומי' with the existing system-prompt addendum, and the UI defaults to that topic so nothing visible changes for existing users until they choose another topic.", - "status": "in-progress", + "status": "done", "priority": "high", "details": "DB migration (shira-hermes side, against insurance_kb DB):\n- New table kb_topic: id PK, slug TEXT UNIQUE, name TEXT NOT NULL, description TEXT, system_prompt_addendum TEXT, is_active BOOLEAN DEFAULT true, created_at TIMESTAMPTZ DEFAULT now(), updated_at TIMESTAMPTZ DEFAULT now().\n- Seed: INSERT (id=1, slug='national-insurance', name='ביטוח לאומי', system_prompt_addendum=, is_active=true). RENAME the DB itself? Probably leave as 'insurance_kb' — costly to rename, no real value. Document in memory.\n- ALTER TABLE kb_source ADD COLUMN topic_id INTEGER REFERENCES kb_topic(id). Backfill: UPDATE kb_source SET topic_id = 1. Then ALTER COLUMN SET NOT NULL.\n\nshira-hermes endpoints:\n- New GET /kb/topics — public, returns active topics with id, slug, name, description (no prompt).\n- /kb/search: accepts optional topic_id (single int). When present, the SQL adds AND s.topic_id = $N. When absent (back-compat), still returns all (might want to deprecate this and require topic_id later).\n- /kb/sources, /kb/source/{id}/chunks, /kb/source/{id}/pdf: same — optional topic filter on listings, no change to single-source endpoints.\n- /kb/ask + /kb/ask/stream: accept topic_id param; load topic.system_prompt_addendum and inject into system_prompt instead of the hardcoded insurance text in _build_ask_runner_context. Pin the legal_kb tool to topic_id too (so search_insurance_kb queries are constrained — RENAME the tool to search_legal_kb and pass topic_id at registration time).\n- _expand_query in kb_search.py: parametrize the prompt by topic name (currently hardcoded 'הביטוח הלאומי בישראל'). Pass topic.name in.\n\nKnowledgeBase extension (EspoCRM):\n- New route+action GET /KnowledgeBase/action/topics → proxies /kb/topics.\n- Service.search/ask gain topic_id, controller passes through.\n- Template gets a topic + +
+ + +
+
+ + +
+
+ מטא-דאטה נוסף (אופציונלי) +
+
+ + +
+
+ + + + +
+
+ + +
+
+
+
+ + מקסימום 50MB. PDF / DOCX / TXT. +
+ + + +
+

משימות אחרונות

+
+
טוען…
+
+
+ + {{/ifEqual}} +
diff --git a/files/client/custom/modules/knowledge-base/src/views/kb/index.js b/files/client/custom/modules/knowledge-base/src/views/kb/index.js index 5d27511..22a7ff1 100644 --- a/files/client/custom/modules/knowledge-base/src/views/kb/index.js +++ b/files/client/custom/modules/knowledge-base/src/views/kb/index.js @@ -12,6 +12,9 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { let _lastSearch = null; // {query, kind, topicId, hits, selectedIdx, completedAt} let _topics = null; // [{id, slug, name, description, is_active}] let _topicsPromise = null; // dedupe concurrent fetches across mounts + let _activeJobId = null; // currently-polled ingest job id (across remounts) + let _activeJobTopicId = null; // topic this job belongs to (for cross-topic guard) + let _pollIntervalId = null; // setInterval handle so remounts don't double-poll const SS_ASK = 'kb-last-ask'; const SS_SEARCH = 'kb-last-search'; @@ -117,6 +120,10 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { if (!Number.isFinite(id) || id === this.topicId) return; this.switchTopic(id); }, + 'click [data-action="submitUpload"]': function (e) { + e.preventDefault(); + this.runUpload(); + }, }, // Cancel any in-flight ask/search, drop the cached last-result for @@ -190,6 +197,214 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { }); }, + // Phase 2: lazy-loads the recent-jobs list and resumes polling of any + // in-flight upload job. No-op if .kb-jobs-list isn't in the DOM (i.e. + // the user isn't currently on the 'manage' tab). + _ensureManagePanelLoaded: function () { + const self = this; + const $list = this.$el.find('.kb-jobs-list'); + if (!$list.length || this.topicId == null) return; + + // Resume polling if we navigated away mid-upload and came back. + if (_activeJobId && _activeJobTopicId === this.topicId) { + this._startJobPolling(_activeJobId); + } else if (_activeJobId && _activeJobTopicId !== this.topicId) { + // Cross-topic guard: an upload in another topic is invisible + // here. The polling continues silently in the background. + } + + // Always refresh the recent-jobs list on mount. + this._loadJobsList(); + }, + + _loadJobsList: function () { + const self = this; + const $list = this.$el.find('.kb-jobs-list'); + if (!$list.length) return; + const topicAtFetch = this.topicId; + Espo.Ajax.getRequest('KnowledgeBase/action/jobs', { + topicId: topicAtFetch, + limit: 20, + }).then(res => { + if (self.topicId !== topicAtFetch) return; // user switched + self._renderJobsList(res && res.items || []); + }).catch(err => { + console.error('KB: failed to load jobs', err); + $list.html('
שגיאה בטעינת רשימת המשימות.
'); + }); + }, + + _renderJobsList: function (items) { + const $list = this.$el.find('.kb-jobs-list'); + if (!$list.length) return; + if (!items.length) { + $list.html('
אין משימות עדיין.
'); + return; + } + const statusHe = { + queued: 'בתור', + processing: 'מעובד…', + done: 'הושלם', + failed: 'נכשל', + }; + const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר'}; + const rows = items.map(j => { + const when = j.completed_at || j.started_at || j.created_at || ''; + const whenShort = String(when).slice(0, 19).replace('T', ' '); + const meta = j.metadata_json || {}; + const title = this.escape(meta.title || j.original_filename || '—'); + const fname = this.escape(j.original_filename || ''); + const detail = j.status === 'done' + ? `${j.chunks_created || 0} קטעים` + : (j.status === 'failed' + ? `${this.escape((j.error_message || '').slice(0, 80))}` + : ''); + return `
+
+
${title}
+
+ ${kindHe[j.kind] || j.kind} · ${this.escape(whenShort)}${j.requested_by_user ? ' · ' + this.escape(j.requested_by_user) : ''} +
+ ${detail ? '
' + detail + '
' : ''} +
+
${statusHe[j.status] || this.escape(j.status)}
+
`; + }).join(''); + $list.html(rows); + }, + + // Live status panel rendered above the upload form during processing. + _renderJobStatus: function (job) { + const $form = this.$el.find('.kb-upload-form'); + if (!$form.length) return; + // Remove any previous banner so we can replace it. + this.$el.find('.kb-upload-status').remove(); + + const statusHe = { + queued: 'בתור', + processing: 'מעובד…', + done: 'הושלם בהצלחה', + failed: 'נכשל', + }; + const klass = job.status === 'done' ? 'alert-success' + : job.status === 'failed' ? 'alert-danger' + : 'alert-info'; + const filename = this.escape(job.original_filename || ''); + const extra = job.status === 'done' + ? ` · נוצרו ${job.chunks_created || 0} קטעים` + : (job.status === 'failed' + ? ` · ${this.escape((job.error_message || '').slice(0, 200))}` + : ''); + const html = `
+ ${this.escape(filename)} — ${statusHe[job.status] || this.escape(job.status)}${extra} +
`; + $form.before(html); + }, + + _startJobPolling: function (jobId) { + const self = this; + this._stopJobPolling(); + _activeJobId = jobId; + _activeJobTopicId = this.topicId; + + const tick = () => { + Espo.Ajax.getRequest('KnowledgeBase/action/job', {id: jobId}) + .then(job => { + if (!self.$el || !self.$el.length) return; + self._renderJobStatus(job); + if (job.status === 'done' || job.status === 'failed') { + self._stopJobPolling(); + _activeJobId = null; + _activeJobTopicId = null; + // Refresh the recent-jobs list to surface the new row. + self._loadJobsList(); + } + }) + .catch(err => { + console.error('KB: job poll failed', err); + // Don't kill the interval on a single transient error; + // let the next tick try again. + }); + }; + tick(); // immediate first read so the user sees status fast + _pollIntervalId = setInterval(tick, 2000); + }, + + _stopJobPolling: function () { + if (_pollIntervalId) { + clearInterval(_pollIntervalId); + _pollIntervalId = null; + } + }, + + runUpload: function () { + const $form = this.$el.find('.kb-upload-form'); + const $file = $form.find('input[type="file"]'); + const file = ($file[0] && $file[0].files && $file[0].files[0]) || null; + if (!file) { + alert('יש לבחור קובץ.'); + return; + } + if (file.size > 50 * 1024 * 1024) { + alert('הקובץ גדול מ-50MB. אנא העלה קובץ קטן יותר.'); + return; + } + if (this.topicId == null) { + alert('נושא לא נבחר.'); + return; + } + + const fd = new FormData(); + fd.append('file', file, file.name); + fd.append('kind', $form.find('select[name="kind"]').val() || 'circular'); + fd.append('topicId', String(this.topicId)); + ['title', 'identifier', 'published_at', 'effective_at', 'source_url'].forEach(name => { + const v = ($form.find(`[name="${name}"]`).val() || '').trim(); + if (v) fd.append(name, v); + }); + + const $btn = this.$el.find('[data-action="submitUpload"]'); + $btn.prop('disabled', true).text('מעלה…'); + + const self = this; + // Bypass Espo.Ajax.postRequest (which JSON-encodes the body) — use + // raw $.ajax so the FormData multipart body is sent intact. + $.ajax({ + url: 'api/v1/KnowledgeBase/action/upload', + method: 'POST', + data: fd, + contentType: false, + processData: false, + cache: false, + }).then(res => { + $btn.prop('disabled', false).text('העלה'); + // Reset the form so the user can upload another. + $form[0].reset(); + if (res && res.job_id) { + self._renderJobStatus({ + original_filename: file.name, + status: res.status || 'queued', + }); + self._startJobPolling(res.job_id); + } + }).fail(xhr => { + $btn.prop('disabled', false).text('העלה'); + let msg = 'שגיאה לא ידועה'; + try { + const json = JSON.parse(xhr.responseText || '{}'); + msg = json.message || json.detail || xhr.responseText || msg; + } catch (e) { + msg = xhr.responseText || msg; + } + self.$el.find('.kb-upload-status').remove(); + self.$el.find('.kb-upload-form').before( + '
' + + 'העלאה נכשלה: ' + self.escape(msg.slice(0, 300)) + + '
' + ); + }); + }, + _ensureTopicsLoaded: function () { if (_topics) return Promise.resolve(_topics); if (_topicsPromise) return _topicsPromise; @@ -253,6 +468,13 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { _clear(sessionStorage, SS_SEARCH); this._sources = null; this._sourcesPromise = null; + // An in-flight upload belongs to the topic where it started; + // stop polling it here so the new topic doesn't see a stale banner. + // The job continues processing on the server; the user can find + // it under "משימות אחרונות" if they switch back. + this._stopJobPolling(); + _activeJobId = null; + _activeJobTopicId = null; this.stopAskProgress(); this.setLoading(false); this.$el.find('.kb-results').empty(); @@ -272,6 +494,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { if (!self.$el || !self.$el.length) return; self._populateTopicPicker(); self._ensureSourcesLoaded(); + self._ensureManagePanelLoaded(); }; if (_topics) { afterTopics(); diff --git a/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php b/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php index 48d7b03..9b9b5f0 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php +++ b/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php @@ -121,4 +121,74 @@ class KnowledgeBase $this->coerceTopicId($data->topicId ?? $data->topic_id ?? null) ); } + + /** + * Multipart upload to shira-hermes /admin/kb/upload. We rely on PHP's + * automatic multipart parsing via $_FILES because the EspoCRM Request + * interface doesn't expose uploaded files directly. The user's + * EspoCRM identity is forwarded as `X-User-Name` so the upstream job + * row records who uploaded what. + */ + public function postActionUpload(Request $request, Response $response): array + { + $this->checkAccess(); + + $files = $_FILES['file'] ?? null; + if (!$files || ($files['error'] ?? UPLOAD_ERR_NO_FILE) !== UPLOAD_ERR_OK) { + throw new BadRequest('No file uploaded (form field must be `file`).'); + } + $tmpPath = $files['tmp_name'] ?? ''; + if (!$tmpPath || !is_uploaded_file($tmpPath)) { + throw new BadRequest('Invalid upload — tmp_name missing.'); + } + + // For multipart/form-data EspoCRM's getParsedBody() returns an empty + // stdClass; the form fields land in $_POST as PHP parses them. + $kind = $_POST['kind'] ?? null; + if (!in_array($kind, ['law', 'regulation', 'circular'], true)) { + throw new BadRequest('kind must be one of: law, regulation, circular.'); + } + $topicId = $this->coerceTopicId($_POST['topicId'] ?? $_POST['topic_id'] ?? null); + if ($topicId === null) { + throw new BadRequest('topicId is required.'); + } + + $metadata = [ + 'title' => isset($_POST['title']) ? trim((string) $_POST['title']) : '', + 'identifier' => isset($_POST['identifier']) ? trim((string) $_POST['identifier']) : '', + 'published_at' => $_POST['published_at'] ?? $_POST['publishedAt'] ?? '', + 'effective_at' => $_POST['effective_at'] ?? $_POST['effectiveAt'] ?? '', + 'source_url' => $_POST['source_url'] ?? $_POST['sourceUrl'] ?? '', + ]; + + return $this->getService()->uploadFile( + $tmpPath, + (string) ($files['name'] ?? 'upload'), + (string) $kind, + $topicId, + $metadata, + (string) $this->user->get('userName') + ); + } + + public function getActionJobs(Request $request, Response $response): array + { + $this->checkAccess(); + $topicId = $this->coerceTopicId($request->getQueryParam('topicId')); + $status = $request->getQueryParam('status'); + $limit = $request->getQueryParam('limit'); + $limitInt = ($limit !== null && is_numeric($limit)) ? (int) $limit : 50; + + return $this->getService()->listJobs($topicId, $status, $limitInt); + } + + public function getActionJob(Request $request, Response $response): array + { + $this->checkAccess(); + $jobId = $request->getQueryParam('id'); + if (!$jobId || !is_numeric($jobId)) { + throw new BadRequest('id (numeric) is required.'); + } + return $this->getService()->getJob((int) $jobId); + } } diff --git a/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json b/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json index f411dbe..2a9241f 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json +++ b/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json @@ -38,5 +38,29 @@ "controller": "KnowledgeBase", "action": "ask" } + }, + { + "route": "/KnowledgeBase/action/upload", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "upload" + } + }, + { + "route": "/KnowledgeBase/action/jobs", + "method": "get", + "params": { + "controller": "KnowledgeBase", + "action": "jobs" + } + }, + { + "route": "/KnowledgeBase/action/job", + "method": "get", + "params": { + "controller": "KnowledgeBase", + "action": "job" + } } ] diff --git a/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php b/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php index bbc7aa9..3d60aa4 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php +++ b/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php @@ -70,6 +70,136 @@ class KnowledgeBaseService return $this->get($path); } + /** + * Phase 2: async upload to /admin/kb/upload. Forwards the temp-uploaded + * file via cURL multipart and the EspoCRM username as `X-User-Name` + * so shira-hermes can record who uploaded what. + * + * @return array{job_id:int, status:string} + */ + public function uploadFile( + string $tmpPath, + string $filename, + string $kind, + int $topicId, + array $metadata, + string $username + ): array { + if (!is_readable($tmpPath)) { + throw new Error('Uploaded file is not readable on disk.'); + } + if (!in_array($kind, ['law', 'regulation', 'circular'], true)) { + throw new BadRequest('kind must be one of: law, regulation, circular.'); + } + + $url = $this->getBaseUrl() . '/admin/kb/upload'; + $apiKey = $this->getApiKey(); + if (!$apiKey) { + throw new Error('SmartAssistant API key is not configured.'); + } + + // Build the multipart form. CURLOPT_POSTFIELDS as an array makes + // cURL set the multipart Content-Type with a generated boundary. + $fields = [ + 'kind' => $kind, + 'topic_id' => (string) $topicId, + 'file' => new \CURLFile($tmpPath, $this->guessMime($filename), $filename), + ]; + foreach (['title', 'identifier', 'published_at', 'effective_at', 'source_url'] as $k) { + $val = $metadata[$k] ?? ''; + if ($val !== '' && $val !== null) { + $fields[$k] = (string) $val; + } + } + + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_POST => true, + CURLOPT_POSTFIELDS => $fields, + CURLOPT_HTTPHEADER => [ + 'Accept: application/json', + 'X-Api-Key: ' . $apiKey, + 'X-User-Name: ' . $username, + ], + CURLOPT_RETURNTRANSFER => true, + CURLOPT_TIMEOUT => 120, + CURLOPT_CONNECTTIMEOUT => 10, + ]); + + $body = curl_exec($ch); + $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + $err = curl_error($ch); + curl_close($ch); + + if ($err) { + $this->log->error("KnowledgeBase: upload transport error: {$err}"); + throw new Error("Failed to reach Knowledge Base: {$err}"); + } + if ($httpCode === 413) { + throw new BadRequest('File too large (max 50MB).'); + } + if ($httpCode >= 400 && $httpCode < 500) { + $detail = $this->decodeDetail($body) ?: 'Bad request'; + throw new BadRequest($detail); + } + if ($httpCode < 200 || $httpCode >= 300) { + $this->log->error("KnowledgeBase: upload HTTP {$httpCode}: {$body}"); + throw new Error("Knowledge Base returned HTTP {$httpCode}"); + } + + $decoded = json_decode((string) $body, true); + if (!is_array($decoded) || !isset($decoded['job_id'])) { + throw new Error('Invalid response from Knowledge Base.'); + } + return $decoded; + } + + public function listJobs(?int $topicId, ?string $status, int $limit): array + { + $limit = max(1, min(200, $limit)); + $qs = ['limit' => $limit]; + if ($topicId !== null) { + $qs['topic_id'] = $topicId; + } + if ($status !== null && $status !== '') { + $qs['status'] = $status; + } + $path = '/admin/kb/jobs?' . http_build_query($qs); + return $this->get($path); + } + + public function getJob(int $jobId): array + { + return $this->get('/admin/kb/jobs/' . $jobId); + } + + private function guessMime(string $filename): string + { + $lower = strtolower($filename); + if (str_ends_with($lower, '.pdf')) { + return 'application/pdf'; + } + if (str_ends_with($lower, '.docx')) { + return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + } + if (str_ends_with($lower, '.txt')) { + return 'text/plain'; + } + return 'application/octet-stream'; + } + + private function decodeDetail($body): ?string + { + if (!is_string($body)) { + return null; + } + $decoded = json_decode($body, true); + if (is_array($decoded) && isset($decoded['detail'])) { + return is_string($decoded['detail']) ? $decoded['detail'] : json_encode($decoded['detail']); + } + return null; + } + public function ask( string $message, ?string $conversationId, diff --git a/manifest.json b/manifest.json index 1cce512..8c4f82a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "name": "KnowledgeBase", "module": "KnowledgeBase", - "version": "0.3.0", + "version": "0.4.0", "acceptableVersions": [ ">=8.0.0" ],