diff --git a/.taskmaster/tasks/tasks.json b/.taskmaster/tasks/tasks.json index d1e055d..5daa24a 100644 --- a/.taskmaster/tasks/tasks.json +++ b/.taskmaster/tasks/tasks.json @@ -180,7 +180,7 @@ "id": "15", "title": "Phase 4 — Topic CRUD UI (admins can add/edit/disable topics from the panel)", "description": "Once Phase 1 ships there will be a single seeded topic ('ביטוח לאומי'). For Klear and other firms to actually use the multi-topic capability they need a UI to create new topics — דיני עבודה, דין פלילי, נדל\"ן — without anyone running SQL. Add a topics-management section in the 'ניהול' tab: list active+inactive topics, add new (slug + name + system_prompt_addendum), edit (rename, change prompt addendum), soft-delete (sets is_active=false). Soft-delete keeps the data but hides the topic from the user-facing dropdown.", - "status": "in-progress", + "status": "done", "priority": "normal", "details": "shira-hermes endpoints:\n- GET /admin/kb/topics: list including inactive ones (with source counts).\n- POST /admin/kb/topics: create. Validates slug (lowercase, hyphens only, unique). Returns created row.\n- PUT /admin/kb/topics/{id}: update name, description, system_prompt_addendum, is_active.\n- DELETE /admin/kb/topics/{id}: 409 Conflict if topic has sources; otherwise hard-delete. Soft-delete (is_active=false) is the normal path.\n\nEspoCRM client:\n- New section in 'ניהול' tab: 'נושאים' table — slug, name, source count, is_active toggle, edit button.\n- Add new: modal with slug, name, description, prompt-addendum textarea (pre-populated with a generic template like 'You are answering legal questions about . Cite sections explicitly when possible. Never answer from generic legal training if the KB has a matching section').\n- Edit: same modal pre-filled. Saving updates the topic; if name changed, the topic dropdown refreshes.\n- The 'ניהול' tab itself only shows the topics section if user is admin. Regular users see only sources management for the topics they have access to.\n\nDefault topic on first install of multi-topic version: still id=1 'ביטוח לאומי'. New installs get the same seed.", "testStrategy": "1) Create topic 'דיני עבודה' (slug 'employment-law'). 2) Verify it appears in GET /kb/topics within seconds. 3) The user-facing topic dropdown lists it. 4) Upload a PDF under it (Phase 2 flow), verify search/ask scoped to it works. 5) Edit prompt addendum — /kb/ask answers in context of employment law. 6) Soft-delete the test topic — disappears from user dropdown but kept in admin list.", @@ -189,7 +189,7 @@ "12" ], "createdAt": "2026-04-25T13:30:00Z", - "updatedAt": "2026-04-25T17:30:11.758Z" + "updatedAt": "2026-04-25T17:47:57.615Z" }, { "id": "17", @@ -207,16 +207,17 @@ "id": "16", "title": "Phase 5 (optional) — Per-topic ACL by EspoCRM role", "description": "Not every user in a multi-topic firm should see every topic. Example: only the criminal-law department needs the criminal KB; the family-law team shouldn't. Wire EspoCRM's existing role/team model to the KB topics: each topic can be restricted to specific roles, with two permission levels — 'view' (use search/ask) and 'manage' (upload/edit/delete). Without role mapping, all non-portal users see all topics (current behavior). With mapping, the topic dropdown filters to topics the user has at least 'view' on, and the management UI gates 'manage' actions to users with the manage permission for that topic.", - "status": "pending", + "status": "deferred", "priority": "low", - "details": "DB: new table kb_topic_acl: topic_id FK, role_name TEXT, permission TEXT CHECK in ('view', 'manage'), PRIMARY KEY (topic_id, role_name, permission).\n\nshira-hermes endpoints:\n- /kb/topics: filter by user's roles. The user's role list comes from the EspoCRM proxy (sent in a header or JWT claim — needs design).\n- /admin/kb/* actions: enforce 'manage' permission per source's topic_id.\n\nEspoCRM client:\n- Topic dropdown: only shows topics the user has 'view' on.\n- Management tab: only visible if user has 'manage' on at least one topic.\n- Edit/delete buttons in the sources table: visible only when user has 'manage' on that source's topic.\n\nThis is the most-likely-to-skip phase if the user's firm is small and everyone needs everything. Worth scoping out depth before committing.", + "details": "DB: new table kb_topic_acl: topic_id FK, role_name TEXT, permission TEXT CHECK in ('view', 'manage'), PRIMARY KEY (topic_id, role_name, permission).\n\nshira-hermes endpoints:\n- /kb/topics: filter by user's roles. The user's role list comes from the EspoCRM proxy (sent in a header or JWT claim — needs design).\n- /admin/kb/* actions: enforce 'manage' permission per source's topic_id.\n\nEspoCRM client:\n- Topic dropdown: only shows topics the user has 'view' on.\n- Management tab: only visible if user has 'manage' on at least one topic.\n- Edit/delete buttons in the sources table: visible only when user has 'manage' on that source's topic.\n\nThis is the most-likely-to-skip phase if the user's firm is small and everyone needs everything. Worth scoping out depth before committing.\n\n---\nDEFERRED 2026-04-25: small firm, all users need access to all topics. Re-open if the firm grows past ~3 lawyers AND topics start to genuinely need siloing (e.g. confidential criminal cases the partners shouldn't browse). Until then, the only access control is `User::isPortal()` already enforced in `Controller::checkAccess()`.", "testStrategy": "Set up two test users: lawyer_a with role 'criminal-law', lawyer_b with role 'employment-law'. Topics: 'דין פלילי' restricted to 'criminal-law' role only, 'דיני עבודה' restricted to 'employment-law'. lawyer_a sees only the criminal topic; lawyer_b only employment. Cross-user attempts return 403.", "subtasks": [], "dependencies": [ "12", "15" ], - "createdAt": "2026-04-25T13:30:00Z" + "createdAt": "2026-04-25T13:30:00Z", + "updatedAt": "2026-04-25T17:51:07.124Z" }, { "id": "18", @@ -241,13 +242,25 @@ "subtasks": [], "dependencies": [], "createdAt": "2026-04-25T16:54:54.478470Z" + }, + { + "id": "20", + "title": "v0.8.0 — kind 'tool' + labels + bulk upload with AI classification", + "description": "Phase 6. Adds 'tool' kind for academic assessment instruments (GMFCS, MACS, etc), label-based sub-topic grouping (kb_label many-to-many), multi-file drag-drop upload, and AI-extracted metadata via Claude/ai-gateway. Two-phase ingest: classify → await_review → embed. Replaces single-file upload with batch-review screen. Also creates 'תוויות' admin tab. Subsumes task #19 (labels). Plan: ~/.claude/plans/hidden-tickling-ullman.md", + "details": "See /home/chaim/.claude/plans/hidden-tickling-ullman.md for the full architecture, migration scripts, API surface, classifier prompt, UX, and file-level breakdown. ~7 days estimated.", + "testStrategy": "", + "status": "in-progress", + "dependencies": [], + "priority": "medium", + "subtasks": [], + "updatedAt": "2026-04-25T18:21:44.731Z" } ], "metadata": { "version": "1.0.0", - "lastModified": "2026-04-25T17:30:11.759Z", - "taskCount": 19, - "completedCount": 12, + "lastModified": "2026-04-25T18:21:44.731Z", + "taskCount": 20, + "completedCount": 13, "tags": [ "master" ] diff --git a/files/client/custom/modules/knowledge-base/res/templates/kb/index.tpl b/files/client/custom/modules/knowledge-base/res/templates/kb/index.tpl index f396bf4..4f2f6a4 100644 --- a/files/client/custom/modules/knowledge-base/res/templates/kb/index.tpl +++ b/files/client/custom/modules/knowledge-base/res/templates/kb/index.tpl @@ -37,6 +37,7 @@ + -
- - -
-
- מטא-דאטה נוסף (אופציונלי) -
-
- - -
-
- - - - -
-
- - -
-
-
-
- - מקסימום 50MB. PDF / DOCX / TXT. -
- + +
+ ניתן לבחור מספר קבצים בו-זמנית. שירה תנתח כל קובץ ותציע מטא-דאטה (סוג / כותרת / תוויות / סיכום) — תוכל לערוך לפני אישור הטמעה. + מקסימום 50MB לקובץ, עד 50 קבצים בקבוצה. +
+
+
diff --git a/files/client/custom/modules/knowledge-base/src/views/kb/index.js b/files/client/custom/modules/knowledge-base/src/views/kb/index.js index 79efd58..3a19291 100644 --- a/files/client/custom/modules/knowledge-base/src/views/kb/index.js +++ b/files/client/custom/modules/knowledge-base/src/views/kb/index.js @@ -17,6 +17,9 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { let _pollIntervalId = null; // setInterval handle so remounts don't double-poll let _adminSourcesByTopic = {}; // {topicId: [...sources]} cache for the manage tab let _adminTopics = null; // [...topics admin view] cache (incl. is_active=false) + let _adminLabels = null; // labels admin cache + let _activeBatch = null; // {batchId, kind, cards:[{jobId, filename, status, ai, edits, labels}]} + let _batchPollIntervalId = null; const SS_ASK = 'kb-last-ask'; const SS_SEARCH = 'kb-last-search'; @@ -190,6 +193,49 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { const id = parseInt($(e.currentTarget).data('id'), 10); if (id) this._deleteTopic(id); }, + // ── Phase 6 (v0.8.0): batch upload + labels admin ────────────── + 'click [data-action="pickBatchFiles"]': function (e) { + e.preventDefault(); + this.$el.find('#kb-batch-files').trigger('click'); + }, + 'change #kb-batch-files': function (e) { + const files = Array.from(e.target.files || []); + if (files.length) this._uploadBatch(files); + e.target.value = ''; + }, + 'click [data-action="discardCard"]': function (e) { + e.preventDefault(); + const jobId = parseInt($(e.currentTarget).data('jobid'), 10); + if (jobId) this._discardCard(jobId); + }, + 'click [data-action="commitAllBatch"]': function (e) { + e.preventDefault(); + this._commitAllBatch(); + }, + 'click [data-action="discardAllBatch"]': function (e) { + e.preventDefault(); + this._discardAllBatch(); + }, + 'click [data-action="refreshLabels"]': function (e) { + e.preventDefault(); + this._loadAdminLabels(/*force*/ true); + }, + 'click [data-action="deleteLabel"]': function (e) { + e.preventDefault(); + const id = parseInt($(e.currentTarget).data('id'), 10); + if (id) this._deleteLabel(id); + }, + 'click [data-action="refreshPending"]': function (e) { + e.preventDefault(); + this._loadPendingBatches(); + }, + 'click [data-action="openBatch"]': function (e) { + e.preventDefault(); + const batchId = $(e.currentTarget).data('batchid'); + const topicAttr = $(e.currentTarget).data('topicid'); + const topicId = (topicAttr === '' || topicAttr == null) ? null : parseInt(topicAttr, 10); + if (batchId) this._openBatch(String(batchId), topicId); + }, }, // Cancel any in-flight ask/search, drop the cached last-result for @@ -285,6 +331,17 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { this._loadAdminSources(); // Phase 4: load the topics admin table (firm-wide, not topic-scoped). this._loadAdminTopics(); + // Phase 6: load the labels admin table. + this._loadAdminLabels(); + // Phase 6: load batches that still need review (RAM-only + // _activeBatch is wiped by hard refresh; this lets the user + // resume any pending batch from the panel). + this._loadPendingBatches(); + // Phase 6: redraw any in-flight batch upload (survives remount). + if (_activeBatch) { + this._renderBatchCards(); + if (this._batchHasClassifying()) this._startBatchPolling(); + } }, _loadJobsList: function () { @@ -317,7 +374,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { done: 'הושלם', failed: 'נכשל', }; - const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר', caselaw: 'פסיקה'}; + const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר', caselaw: 'פסיקה', tool: 'כלי הערכה'}; const rows = items.map(j => { const when = j.completed_at || j.started_at || j.created_at || ''; const whenShort = String(when).slice(0, 19).replace('T', ' '); @@ -618,7 +675,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { return; } - const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר', caselaw: 'פסיקה'}; + const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזר', caselaw: 'פסיקה', tool: 'כלי הערכה'}; const rows = items.map(s => { const li = s.last_ingest || {}; const failedFlag = li.status === 'failed' @@ -772,6 +829,506 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { }); }, + // ── Phase 6 (v0.8.0): labels admin ───────────────────────────────── + + _loadAdminLabels: function (force) { + const $table = this.$el.find('.kb-labels-table'); + if (!$table.length) return; + if (!force && _adminLabels) { + this._renderLabelsTable(); + return; + } + const self = this; + Espo.Ajax.getRequest('KnowledgeBase/action/labels', { + topicId: this.topicId, + limit: 200, + }).then(res => { + _adminLabels = res && res.items || []; + self._renderLabelsTable(); + }).catch(err => { + console.error('KB: load labels failed', err); + $table.html('
שגיאה בטעינת תוויות.
'); + }); + }, + + _renderLabelsTable: function () { + const $table = this.$el.find('.kb-labels-table'); + if (!$table.length) return; + const items = _adminLabels || []; + if (!items.length) { + $table.html('
אין תוויות עדיין. הן ייווצרו אוטומטית מההצעות של שירה בעת ההעלאה.
'); + return; + } + const rows = items.map(l => ` + ${this.escape(l.name)} + ${this.escape(l.slug)} + ${l.usage_count || 0} + + ${(l.usage_count || 0) === 0 + ? `` + : 'בשימוש'} + + `).join(''); + $table.html(` + + + + + ${rows} +
שם תוויתslugשימושים
+ `); + }, + + _deleteLabel: function (id) { + const lbl = (_adminLabels || []).find(l => l.id === id); + if (!lbl) return; + if (!confirm(`למחוק את התווית "${lbl.name}" (${lbl.slug})?`)) return; + const self = this; + Espo.Ajax.postRequest('KnowledgeBase/action/deleteLabel', {id}) + .then(() => { + _adminLabels = (_adminLabels || []).filter(l => l.id !== id); + self._renderLabelsTable(); + }) + .catch(err => { + let msg = 'שגיאה לא ידועה'; + try { const j = JSON.parse(err && err.responseText || '{}'); msg = j.message || j.detail || msg; } catch (e) {} + alert('מחיקה נכשלה: ' + String(msg).slice(0, 200)); + }); + }, + + // ── Phase 6: batch upload + AI review ────────────────────────────── + + // Pending-review panel: resumes any batch the user uploaded + // earlier and didn't commit. _activeBatch is RAM-only, so a hard + // refresh wipes the review screen — without this list the only + // recovery path was the DB. + _loadPendingBatches: function () { + const self = this; + const $list = this.$el.find('.kb-pending-list'); + const $count = this.$el.find('.kb-pending-count'); + if (!$list.length) return; + Espo.Ajax.getRequest('KnowledgeBase/action/pendingBatches', {limit: 50}) + .then(res => { + const items = (res && res.items) || []; + self._renderPendingBatches(items); + $count.text(items.length ? ` (${items.length})` : ''); + }) + .catch(err => { + console.error('KB: failed to load pending batches', err); + $list.html('
שגיאה בטעינת ממתינים לאישור.
'); + }); + }, + + _renderPendingBatches: function (items) { + const $list = this.$el.find('.kb-pending-list'); + if (!$list.length) return; + if (!items.length) { + $list.html('
אין באצ׳ים ממתינים לאישור.
'); + return; + } + const kindHe = { + law: 'חוק', regulation: 'תקנות', circular: 'חוזרים', + caselaw: 'פסיקה', tool: 'כלי הערכה', + }; + const escape = this.escape.bind(this); + const html = items.map(b => { + const kinds = (b.kinds || []).map(k => kindHe[k] || k).join(', '); + const time = b.created_at ? new Date(b.created_at).toLocaleString('he-IL') : ''; + const counters = []; + if (b.awaiting_review) counters.push(`${b.awaiting_review} מוכנים`); + if (b.processing) counters.push(`${b.processing} בתהליך`); + if (b.queued) counters.push(`${b.queued} בתור`); + if (b.failed) counters.push(`${b.failed} נכשלו`); + if (b.done) counters.push(`${b.done} הושלמו`); + const isCurrent = _activeBatch && _activeBatch.batchId === b.batch_id; + const more = b.total > 1 ? ` + ${b.total - 1} נוספים` : ''; + return `
+
+
📦 ${escape(b.first_filename || '(ללא שם)')}${more}
+
${escape(kinds)} · ${escape(time)} · ${counters.join(' ')}
+
+ +
`; + }).join(''); + $list.html(html); + }, + + _openBatch: function (batchId, topicId) { + const self = this; + if (!batchId) return; + if (_activeBatch && _activeBatch.batchId === batchId) { + this._scrollToBatch(); + return; + } + // If the batch belongs to a different topic, switch — labels + // and source views are topic-scoped, the user should see the + // matching context. + if (topicId && this.topicId !== topicId) { + this.switchTopic(topicId); + } + this._stopBatchPolling(); + Espo.Ajax.getRequest('KnowledgeBase/action/batch', {batchId}) + .then(res => { + const items = (res && res.items) || []; + if (!items.length) { + alert('הבאצ׳ ריק או נמחק.'); + return; + } + const cards = items.map(it => { + let ai = it.ai_suggestions; + if (typeof ai === 'string') { + try { ai = JSON.parse(ai); } catch (e) { ai = {}; } + } + let s = it.status; + if (s === 'processing' && it.processing_stage === 'classifying') s = 'classifying'; + else if (s === 'processing' && it.processing_stage === 'embedding') s = 'committing'; + return { + jobId: it.id, + filename: it.original_filename, + status: s, + ai: (ai && Object.keys(ai).length) ? ai : null, + edits: null, + error: it.error_message || null, + }; + }); + _activeBatch = { + batchId: batchId, + kind: (items[0] && items[0].kind) || 'circular', + topicId: topicId || self.topicId, + cards, + }; + self._renderBatchCards(); + // Repaint pending list so the opened row shows "פתוח" instead of "פתח". + self._loadPendingBatches(); + self._scrollToBatch(); + if (self._batchHasClassifying() || self._batchHasCommitting()) { + self._startBatchPolling(); + } + }) + .catch(err => { + console.error('KB: failed to open batch', err); + alert('פתיחת הבאצ׳ נכשלה.'); + }); + }, + + _scrollToBatch: function () { + const $batch = this.$el.find('.kb-batch-panel'); + if ($batch.length && $batch[0].scrollIntoView) { + $batch[0].scrollIntoView({behavior: 'smooth', block: 'start'}); + } + }, + + _uploadBatch: function (fileList) { + const $kindSel = this.$el.find('#kb-batch-kind'); + const kind = $kindSel.val() || 'circular'; + const topicId = this.topicId; + if (topicId == null) { + alert('בחר נושא לפני העלאה.'); + return; + } + // Filter on size + extension client-side; server enforces too. + const ok = []; + for (const f of fileList) { + if (!/\.(pdf|docx|txt)$/i.test(f.name)) continue; + if (f.size > 50 * 1024 * 1024) { + alert(`הקובץ "${f.name}" גדול מ-50MB ולא יועלה.`); + continue; + } + ok.push(f); + } + if (!ok.length) return; + + const fd = new FormData(); + fd.append('kind', kind); + fd.append('topicId', String(topicId)); + // PHP-side $_FILES['files'] requires the bracket suffix to + // recognize repeated fields as a multi-file array. The + // EspoCRM controller un-array's into a plain `files` field + // before forwarding to shira-hermes (FastAPI standard). + for (const f of ok) fd.append('files[]', f, f.name); + + // Initialize cards in placeholder state. They get the real + // job_id from the server response. + _activeBatch = { + batchId: null, + kind, + topicId, + cards: ok.map(f => ({ + jobId: null, + filename: f.name, + status: 'uploading', + ai: null, + edits: null, + })), + }; + this._renderBatchCards(); + + const self = this; + $.ajax({ + url: 'api/v1/KnowledgeBase/action/uploadBatch', + method: 'POST', + data: fd, + contentType: false, + processData: false, + cache: false, + timeout: 300000, + }).then(res => { + if (!_activeBatch) return; + _activeBatch.batchId = res.batch_id; + // Match returned jobs to local cards by filename order. + (res.jobs || []).forEach((j, i) => { + if (_activeBatch.cards[i]) { + _activeBatch.cards[i].jobId = j.job_id || null; + _activeBatch.cards[i].status = j.error ? 'failed' : 'queued'; + if (j.error) _activeBatch.cards[i].error = j.error; + } + }); + self._renderBatchCards(); + self._startBatchPolling(); + }).fail(xhr => { + let msg = 'שגיאה לא ידועה'; + try { + const j = JSON.parse(xhr.responseText || '{}'); + msg = j.message || j.detail || xhr.responseText || msg; + } catch (e) { msg = xhr.responseText || msg; } + alert('העלאה נכשלה: ' + String(msg).slice(0, 300)); + _activeBatch = null; + self._renderBatchCards(); + }); + }, + + _renderBatchCards: function () { + const $cards = this.$el.find('.kb-batch-cards'); + const $actions = this.$el.find('.kb-batch-actions'); + if (!$cards.length) return; + if (!_activeBatch || !_activeBatch.cards.length) { + $cards.empty(); + $actions.hide(); + return; + } + + const statusLabel = (s) => ({ + uploading: 'מעלה…', + queued: 'בתור', + classifying: 'מנתח מטא-דאטה…', + awaiting_review: '[✓ מוכן]', + committing: 'מטמיע…', + done: '[✓ הושלם]', + failed: '[⚠️ נכשל]', + }[s] || `${this.escape(s)}`); + + const kindOptions = (selected) => { + const opts = [ + ['law', 'חוק'], + ['regulation', 'תקנות'], + ['circular', 'חוזר'], + ['caselaw', 'פסיקה'], + ['tool', 'כלי הערכה'], + ]; + return opts.map(([v, l]) => + `` + ).join(''); + }; + + let readyCount = 0; + const html = _activeBatch.cards.map((c, idx) => { + if (c.status === 'awaiting_review') readyCount++; + const ai = c.ai || {}; + const e = c.edits || {}; + const val = (k) => this.escape(e[k] != null ? e[k] : (ai[k] || '')); + const labelsStr = e.labelsStr != null ? e.labelsStr : ((ai.labels || []).join(', ')); + let body = ''; + if (c.status === 'failed') { + body = `
${this.escape(c.error || 'שגיאה')}
`; + } else if (c.status === 'awaiting_review' || c.status === 'committing' || c.status === 'done') { + const disabled = c.status !== 'awaiting_review' ? ' disabled' : ''; + body = ` +
+ + +
+
+ + +
+ + + `; + } else { + body = '
ממתין לניתוח…
'; + } + return `
+
+
📄 ${this.escape(c.filename)}
+
+ ${statusLabel(c.status)} + ${c.status !== 'done' && c.status !== 'committing' ? `` : ''} +
+
+
${body}
+
`; + }).join(''); + + $cards.html(html); + $actions.toggle(_activeBatch.cards.length > 0); + this.$el.find('.kb-batch-commit-count').text(readyCount > 0 ? `(${readyCount})` : ''); + this.$el.find('[data-action="commitAllBatch"]').prop('disabled', readyCount === 0); + + // Wire input change → save into card.edits + const self = this; + $cards.find('[data-edit]').off('input.batch change.batch').on('input.batch change.batch', function () { + const idx = parseInt($(this).data('card'), 10); + const k = $(this).data('edit'); + if (_activeBatch && _activeBatch.cards[idx]) { + if (!_activeBatch.cards[idx].edits) _activeBatch.cards[idx].edits = {}; + _activeBatch.cards[idx].edits[k] = $(this).val(); + } + }); + }, + + _batchHasClassifying: function () { + return _activeBatch && _activeBatch.cards.some(c => + c.status === 'queued' || c.status === 'classifying' || c.status === 'uploading' + ); + }, + + _startBatchPolling: function () { + this._stopBatchPolling(); + const self = this; + // Refresh the pending list once at start so a freshly-uploaded + // batch appears there right away (without waiting for the + // batch to quiesce). + this._loadPendingBatches(); + const tick = () => { + if (!_activeBatch || !_activeBatch.batchId) return; + Espo.Ajax.getRequest('KnowledgeBase/action/batch', {batchId: _activeBatch.batchId}) + .then(res => { + if (!_activeBatch) return; + const byId = {}; + for (const it of res.items || []) byId[it.id] = it; + for (const card of _activeBatch.cards) { + if (!card.jobId) continue; + const it = byId[card.jobId]; + if (!it) continue; + // Map server status → card status + let s = it.status; + if (s === 'processing' && it.processing_stage === 'classifying') s = 'classifying'; + else if (s === 'processing' && it.processing_stage === 'embedding') s = 'committing'; + card.status = s; + card.error = it.error_message || null; + // Parse ai_suggestions if string + let ai = it.ai_suggestions; + if (typeof ai === 'string') { + try { ai = JSON.parse(ai); } catch (e) { ai = {}; } + } + if (ai && Object.keys(ai).length) card.ai = ai; + } + self._renderBatchCards(); + if (!self._batchHasClassifying() && !self._batchHasCommitting()) { + self._stopBatchPolling(); + // Refresh sources/jobs/labels list since new ones may have landed. + if (_activeBatch.cards.some(c => c.status === 'done')) { + self._loadJobsList(); + self._loadAdminSources(true); + self._loadAdminLabels(true); + } + // Pending list reflects awaiting_review counters + // — refresh on every batch quiescence so commits + // and discards drop the row out cleanly. + self._loadPendingBatches(); + } + }) + .catch(err => console.error('KB: batch poll failed', err)); + }; + tick(); + _batchPollIntervalId = setInterval(tick, 3000); + }, + + _stopBatchPolling: function () { + if (_batchPollIntervalId) { + clearInterval(_batchPollIntervalId); + _batchPollIntervalId = null; + } + }, + + _batchHasCommitting: function () { + return _activeBatch && _activeBatch.cards.some(c => c.status === 'committing'); + }, + + _commitAllBatch: function () { + if (!_activeBatch) return; + const self = this; + const ready = _activeBatch.cards.filter(c => c.status === 'awaiting_review' && c.jobId); + if (!ready.length) return; + for (const card of ready) { + const ai = card.ai || {}; + const e = card.edits || {}; + const labelsStr = e.labelsStr != null ? e.labelsStr : ((ai.labels || []).join(', ')); + const newLabels = labelsStr + .split(',') + .map(s => s.trim()) + .filter(s => s.length); + const payload = { + id: card.jobId, + kind: e.kind || ai.kind || _activeBatch.kind, + title: (e.title != null ? e.title : (ai.title || card.filename)).trim() || card.filename, + identifier: (e.identifier != null ? e.identifier : (ai.identifier || '')).trim() || null, + published_at: (e.published_at != null ? e.published_at : (ai.published_at || '')) || null, + effective_at: (e.effective_at != null ? e.effective_at : (ai.effective_at || '')) || null, + source_url: (e.source_url != null ? e.source_url : (ai.source_url || '')) || null, + summary: (e.summary != null ? e.summary : (ai.summary || '')) || null, + label_slugs: [], + new_labels: newLabels, + }; + card.status = 'committing'; + Espo.Ajax.postRequest('KnowledgeBase/action/commitJob', payload) + .catch(err => { + console.error('KB: commit failed for job', card.jobId, err); + card.status = 'failed'; + card.error = (err && err.responseText) || 'commit failed'; + self._renderBatchCards(); + }); + } + self._renderBatchCards(); + self._startBatchPolling(); + }, + + _discardCard: function (jobId) { + if (!_activeBatch) return; + const idx = _activeBatch.cards.findIndex(c => c.jobId === jobId); + if (idx < 0) return; + // If never queued (server failed before assigning jobId), just drop locally. + if (!jobId) { + _activeBatch.cards.splice(idx, 1); + if (!_activeBatch.cards.length) _activeBatch = null; + this._renderBatchCards(); + return; + } + const self = this; + Espo.Ajax.postRequest('KnowledgeBase/action/discardJob', {id: jobId}) + .finally(() => { + _activeBatch.cards.splice(idx, 1); + if (!_activeBatch.cards.length) _activeBatch = null; + self._renderBatchCards(); + }); + }, + + _discardAllBatch: function () { + if (!_activeBatch) return; + if (!confirm(`לבטל את כל ${_activeBatch.cards.length} הקבצים בקבוצה?`)) return; + const self = this; + const promises = _activeBatch.cards + .filter(c => c.jobId && c.status !== 'done') + .map(c => Espo.Ajax.postRequest('KnowledgeBase/action/discardJob', {id: c.jobId}) + .catch(() => null)); + Promise.all(promises).finally(() => { + _activeBatch = null; + self._stopBatchPolling(); + self._renderBatchCards(); + self._loadJobsList(); + }); + }, + // Live status panel rendered above the upload form during processing. _renderJobStatus: function (job) { const $form = this.$el.find('.kb-upload-form'); @@ -1319,7 +1876,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { return; } const initialIdx = (typeof selectedIdx === 'number' && hits[selectedIdx]) ? selectedIdx : 0; - const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר', caselaw: 'פסיקה'}; + const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר', caselaw: 'פסיקה', tool: 'כלי'}; const self = this; const isPdfBacked = h => h && (h.kind === 'regulation' || h.kind === 'circular'); @@ -1618,7 +2175,7 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { answerHtml = '
' + this.escape(text || '') + '
'; } - const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר', caselaw: 'פסיקה'}; + const kindHe = {law: 'חוק', regulation: 'תקנה', circular: 'חוזר', caselaw: 'פסיקה', tool: 'כלי'}; // Dedup sources by source_id (first occurrence = most relevant), // but remember every page that came up per source so users can @@ -1758,8 +2315,8 @@ define('modules/knowledge-base/views/kb/index', ['view'], function (Dep) { $list.html('
אין מקורות.
'); return; } - const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזרים', caselaw: 'פסיקה'}; - const grouped = {law: [], regulation: [], circular: [], caselaw: []}; + const kindHe = {law: 'חוק', regulation: 'תקנות', circular: 'חוזרים', caselaw: 'פסיקה', tool: 'כלי הערכה'}; + const grouped = {law: [], regulation: [], circular: [], caselaw: [], tool: []}; this._sources.forEach(s => { if (grouped[s.kind]) grouped[s.kind].push(s); }); diff --git a/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php b/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php index f1356d2..2d07abd 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php +++ b/files/custom/Espo/Modules/KnowledgeBase/Controllers/KnowledgeBase.php @@ -307,4 +307,161 @@ class KnowledgeBase } return $this->getService()->deleteAdminTopic((int) $id); } + + // ── Phase 6 (v0.8.0): bulk upload + AI classifier + labels ────────────── + + public function postActionUploadBatch(Request $request, Response $response): array + { + $this->checkAccess(); + + // Multi-file: $_FILES['files'] is an array-of-arrays in PHP when + // the form submits files[] repeated. Normalize per-file. + $rawFiles = $_FILES['files'] ?? null; + if (!$rawFiles || !is_array($rawFiles['tmp_name'] ?? null)) { + throw new BadRequest('No files uploaded (form field must be `files[]`).'); + } + $files = []; + $count = count($rawFiles['tmp_name']); + for ($i = 0; $i < $count; $i++) { + $err = $rawFiles['error'][$i] ?? UPLOAD_ERR_NO_FILE; + $tmp = $rawFiles['tmp_name'][$i] ?? ''; + if ($err !== UPLOAD_ERR_OK || !$tmp || !is_uploaded_file($tmp)) { + continue; // server-side filter; client filters too + } + $files[] = [ + 'tmp_name' => $tmp, + 'name' => (string) ($rawFiles['name'][$i] ?? 'upload'), + 'type' => (string) ($rawFiles['type'][$i] ?? 'application/octet-stream'), + 'size' => (int) ($rawFiles['size'][$i] ?? 0), + ]; + } + if (!$files) { + throw new BadRequest('No valid files in upload.'); + } + + $kind = $_POST['kind'] ?? null; + if (!in_array($kind, ['law', 'regulation', 'circular', 'caselaw', 'tool'], true)) { + throw new BadRequest('kind must be one of: law, regulation, circular, caselaw, tool.'); + } + $topicId = $this->coerceTopicId($_POST['topicId'] ?? $_POST['topic_id'] ?? null); + if ($topicId === null) { + throw new BadRequest('topicId is required.'); + } + + return $this->getService()->uploadBatch( + $files, + (string) $kind, + $topicId, + (string) $this->user->get('userName'), + ); + } + + public function getActionBatch(Request $request, Response $response): array + { + $this->checkAccess(); + $batchId = $request->getQueryParam('batchId'); + if (!$batchId) { + throw new BadRequest('batchId is required.'); + } + return $this->getService()->getBatch((string) $batchId); + } + + public function getActionPendingBatches(Request $request, Response $response): array + { + $this->checkAccess(); + $limit = (int) ($request->getQueryParam('limit') ?? 50); + if ($limit < 1) $limit = 1; + if ($limit > 200) $limit = 200; + return $this->getService()->listPendingBatches($limit); + } + + public function postActionCommitJob(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + if (!$id || !is_numeric($id)) { + throw new BadRequest('id (numeric) is required.'); + } + $kind = $body->kind ?? null; + if (!in_array($kind, ['law', 'regulation', 'circular', 'caselaw', 'tool'], true)) { + throw new BadRequest('kind must be one of: law, regulation, circular, caselaw, tool.'); + } + + $payload = [ + 'kind' => $kind, + 'title' => isset($body->title) ? trim((string) $body->title) : '', + 'identifier' => isset($body->identifier) ? trim((string) $body->identifier) : null, + 'source_url' => $body->source_url ?? $body->sourceUrl ?? null, + 'published_at' => $body->published_at ?? $body->publishedAt ?? null, + 'effective_at' => $body->effective_at ?? $body->effectiveAt ?? null, + 'summary' => isset($body->summary) ? trim((string) $body->summary) : null, + 'label_slugs' => is_array($body->label_slugs ?? null) ? $body->label_slugs : [], + 'new_labels' => is_array($body->new_labels ?? null) ? $body->new_labels : [], + ]; + + return $this->getService()->commitJob( + (int) $id, $payload, (string) $this->user->get('userName') + ); + } + + public function postActionDiscardJob(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + if (!$id || !is_numeric($id)) { + throw new BadRequest('id (numeric) is required.'); + } + return $this->getService()->discardJob( + (int) $id, (string) $this->user->get('userName') + ); + } + + public function getActionLabels(Request $request, Response $response): array + { + $this->checkAccess(); + $topicId = $this->coerceTopicId($request->getQueryParam('topicId')); + $q = $request->getQueryParam('q'); + $limit = $request->getQueryParam('limit'); + $limitInt = ($limit !== null && is_numeric($limit)) ? (int) $limit : 50; + return $this->getService()->listLabels($topicId, $q, $limitInt); + } + + public function postActionCreateLabel(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + if (empty($body->slug) || empty($body->name)) { + throw new BadRequest('slug and name are required.'); + } + return $this->getService()->createLabel([ + 'slug' => trim((string) $body->slug), + 'name' => trim((string) $body->name), + 'topic_id' => isset($body->topic_id) ? (int) $body->topic_id : null, + ], (string) $this->user->get('userName')); + } + + public function postActionMergeLabels(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + $intoId = $body->into_label_id ?? $body->intoLabelId ?? null; + if (!$id || !is_numeric($id) || !$intoId || !is_numeric($intoId)) { + throw new BadRequest('id and into_label_id (numeric) are required.'); + } + return $this->getService()->mergeLabels((int) $id, (int) $intoId); + } + + public function postActionDeleteLabel(Request $request, Response $response): array + { + $this->checkAccess(); + $body = $request->getParsedBody(); + $id = $body->id ?? null; + if (!$id || !is_numeric($id)) { + throw new BadRequest('id (numeric) is required.'); + } + return $this->getService()->deleteLabel((int) $id); + } } diff --git a/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json b/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json index 4a10c66..6f8820b 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json +++ b/files/custom/Espo/Modules/KnowledgeBase/Resources/routes.json @@ -126,5 +126,77 @@ "controller": "KnowledgeBase", "action": "deleteTopic" } + }, + { + "route": "/KnowledgeBase/action/uploadBatch", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "uploadBatch" + } + }, + { + "route": "/KnowledgeBase/action/batch", + "method": "get", + "params": { + "controller": "KnowledgeBase", + "action": "batch" + } + }, + { + "route": "/KnowledgeBase/action/pendingBatches", + "method": "get", + "params": { + "controller": "KnowledgeBase", + "action": "pendingBatches" + } + }, + { + "route": "/KnowledgeBase/action/commitJob", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "commitJob" + } + }, + { + "route": "/KnowledgeBase/action/discardJob", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "discardJob" + } + }, + { + "route": "/KnowledgeBase/action/labels", + "method": "get", + "params": { + "controller": "KnowledgeBase", + "action": "labels" + } + }, + { + "route": "/KnowledgeBase/action/createLabel", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "createLabel" + } + }, + { + "route": "/KnowledgeBase/action/mergeLabels", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "mergeLabels" + } + }, + { + "route": "/KnowledgeBase/action/deleteLabel", + "method": "post", + "params": { + "controller": "KnowledgeBase", + "action": "deleteLabel" + } } ] diff --git a/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php b/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php index 93eb2af..23128f1 100644 --- a/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php +++ b/files/custom/Espo/Modules/KnowledgeBase/Services/KnowledgeBaseService.php @@ -227,6 +227,155 @@ class KnowledgeBaseService return $this->request('DELETE', '/admin/kb/topics/' . $topicId, null); } + // ── Phase 6 (v0.8.0): bulk upload + AI classifier + labels ────────────── + + /** + * Multi-file upload. Each file is sent to /admin/kb/upload-batch in + * one multipart POST that repeats the `files` field. shira-hermes + * assigns the batch_id and fires per-file classify tasks. + * + * @param array $files + * @return array{batch_id:string, jobs:array} + */ + public function uploadBatch(array $files, string $kind, int $topicId, string $username): array + { + if (!$files) { + throw new BadRequest('No files in batch.'); + } + $url = $this->getBaseUrl() . '/admin/kb/upload-batch'; + $apiKey = $this->getApiKey(); + if (!$apiKey) { + throw new Error('SmartAssistant API key is not configured.'); + } + + $fields = [ + 'kind' => $kind, + 'topic_id' => (string) $topicId, + ]; + // PHP cURL accepts repeated form fields by giving the array under + // a special "[]" suffix syntax — but actually CURLOPT_POSTFIELDS + // only sees the LAST value when the key repeats. Workaround: + // build the multipart body manually so we can repeat `files`. + $boundary = '----shira-hermes-' . bin2hex(random_bytes(8)); + $body = ''; + foreach ($fields as $k => $v) { + $body .= "--{$boundary}\r\n"; + $body .= "Content-Disposition: form-data; name=\"{$k}\"\r\n\r\n"; + $body .= $v . "\r\n"; + } + foreach ($files as $f) { + $contents = @file_get_contents($f['tmp_name']); + if ($contents === false) { + throw new Error("Failed to read uploaded file: " . $f['name']); + } + $name = addslashes($f['name']); + $type = $f['type'] ?: $this->guessMime($f['name']); + $body .= "--{$boundary}\r\n"; + $body .= "Content-Disposition: form-data; name=\"files\"; filename=\"{$name}\"\r\n"; + $body .= "Content-Type: {$type}\r\n\r\n"; + $body .= $contents . "\r\n"; + } + $body .= "--{$boundary}--\r\n"; + + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_POST => true, + CURLOPT_POSTFIELDS => $body, + CURLOPT_HTTPHEADER => [ + 'Accept: application/json', + 'Content-Type: multipart/form-data; boundary=' . $boundary, + 'X-Api-Key: ' . $apiKey, + 'X-User-Name: ' . $username, + ], + CURLOPT_RETURNTRANSFER => true, + // Batch upload of e.g. 10×10MB files is dominated by PHP→Python + // network transfer; bump from the single-file 120s. + CURLOPT_TIMEOUT => 300, + CURLOPT_CONNECTTIMEOUT => 10, + ]); + $resp = curl_exec($ch); + $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + $err = curl_error($ch); + curl_close($ch); + + if ($err) { + $this->log->error("KnowledgeBase: batch upload transport error: {$err}"); + throw new Error("Failed to reach Knowledge Base: {$err}"); + } + if ($httpCode === 413) { + throw new BadRequest('Total upload too large.'); + } + if ($httpCode >= 400 && $httpCode < 500) { + $detail = $this->decodeDetail($resp) ?: 'Bad request'; + throw new BadRequest($detail); + } + if ($httpCode < 200 || $httpCode >= 300) { + $this->log->error("KnowledgeBase: batch upload HTTP {$httpCode}: {$resp}"); + throw new Error("Knowledge Base returned HTTP {$httpCode}"); + } + + $decoded = json_decode((string) $resp, true); + if (!is_array($decoded) || !isset($decoded['batch_id'])) { + throw new Error('Invalid response from Knowledge Base.'); + } + return $decoded; + } + + public function getBatch(string $batchId): array + { + return $this->get('/admin/kb/batch/' . rawurlencode($batchId)); + } + + public function listPendingBatches(int $limit): array + { + return $this->get('/admin/kb/batches?' . http_build_query(['limit' => $limit])); + } + + public function commitJob(int $jobId, array $payload, string $username): array + { + return $this->postWithUser( + '/admin/kb/jobs/' . $jobId . '/commit', + $payload, + $username, + ); + } + + public function discardJob(int $jobId, string $username): array + { + return $this->postWithUser( + '/admin/kb/jobs/' . $jobId . '/discard', + null, + $username, + ); + } + + public function listLabels(?int $topicId, ?string $q, int $limit): array + { + $qs = ['limit' => $limit]; + if ($topicId !== null) $qs['topic_id'] = $topicId; + if ($q !== null && $q !== '') $qs['q'] = $q; + return $this->get('/admin/kb/labels?' . http_build_query($qs)); + } + + public function createLabel(array $payload, string $username): array + { + return $this->postWithUser('/admin/kb/labels', $payload, $username); + } + + public function mergeLabels(int $labelId, int $intoLabelId): array + { + return $this->request( + 'POST', + '/admin/kb/labels/' . $labelId . '/merge', + ['into_label_id' => $intoLabelId], + ); + } + + public function deleteLabel(int $labelId): array + { + return $this->request('DELETE', '/admin/kb/labels/' . $labelId, null); + } + private function postWithUser(string $path, ?array $payload, string $username): array { $url = $this->getBaseUrl() . $path; diff --git a/manifest.json b/manifest.json index 5aff018..d1722e9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "name": "KnowledgeBase", "module": "KnowledgeBase", - "version": "0.7.0", + "version": "0.8.0", "acceptableVersions": [ ">=8.0.0" ],