feat: KnowledgeBase extension v0.1.6 — inline PDF viewer in browse tab

KnowledgeBase extension for EspoCRM: UI on top of the shira-hermes KB of Israeli National Insurance law, regulations, and circulars. Three modes: - חיפוש — Hybrid (pgvector + tsvector) + Voyage rerank-2.5, returns ranked chunks with heading_path and citations. - שאל את שירה — Full agent loop; Shira picks up search_insurance_kb as needed and returns a summary with citations. - עיון — Browse all active sources. Click a source: - PDF source (ספר הליקויים, ספר המבחנים, circulars): renders the original PDF inline via an iframe proxied through the KnowledgeBasePdf EntryPoint, so the layout/columns/tables are preserved and Ctrl+F works natively. - TXT source (חוק הביטוח הלאומי scraped from Wikisource): falls back to the hierarchical chunk list with RTL styling. Architecture: - Controller: KnowledgeBase.php — thin proxy to shira-hermes /kb/*. - Service: KnowledgeBaseService.php — shared curl plumbing; derives the shira-hermes base URL from the SmartAssistant integration record so there is no second admin config. - EntryPoint: KnowledgeBasePdf.php — streams the PDF inline, wraps the body in a php://temp stream for setBody, applies a locked-down CSP. - JS: views/kb/index.js branches on source.original_path; modes wired through the SmartAssistant fa_IR i18n convention. Auth model: - Browser → EspoCRM: session cookie / X-Api-Key (EspoCRM's existing auth). - EspoCRM → shira-hermes: X-Api-Key from the SmartAssistant integration (never exposed to the browser). - Portal users are blocked at both the Controller and the EntryPoint. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 11:57:23 +00:00
commit 8829a2e93a
21 changed files with 1011 additions and 0 deletions
@@ -0,0 +1,94 @@
+<?php
+
+namespace Espo\Modules\KnowledgeBase\Controllers;
+
+use Espo\Core\Api\Request;
+use Espo\Core\Api\Response;
+use Espo\Core\Exceptions\BadRequest;
+use Espo\Core\Exceptions\Forbidden;
+use Espo\Core\InjectableFactory;
+use Espo\Core\Acl;
+use Espo\Modules\KnowledgeBase\Services\KnowledgeBaseService;
+use Espo\Entities\User;
+
+/**
+ * Thin proxy controller. The EspoCRM UI talks to these actions; each
+ * action forwards to shira-hermes (/kb/*) with the shared API key held
+ * server-side. The browser never sees the shira-hermes key.
+ */
+class KnowledgeBase
+{
+    private InjectableFactory $injectableFactory;
+    private Acl $acl;
+    private User $user;
+
+    public function __construct(InjectableFactory $injectableFactory, Acl $acl, User $user)
+    {
+        $this->injectableFactory = $injectableFactory;
+        $this->acl = $acl;
+        $this->user = $user;
+    }
+
+    private function getService(): KnowledgeBaseService
+    {
+        return $this->injectableFactory->create(KnowledgeBaseService::class);
+    }
+
+    private function checkAccess(): void
+    {
+        // Anyone with a portal-less account (regular users) can read the KB.
+        if ($this->user->isPortal()) {
+            throw new Forbidden('Portal users have no KB access.');
+        }
+    }
+
+    public function postActionSearch(Request $request, Response $response): array
+    {
+        $this->checkAccess();
+        $data = $request->getParsedBody();
+
+        if (empty($data->query)) {
+            throw new BadRequest('query is required.');
+        }
+
+        return $this->getService()->search(
+            trim($data->query),
+            $data->kind ?? 'any',
+            (int) ($data->top_k ?? $data->topK ?? 8)
+        );
+    }
+
+    public function getActionSources(Request $request, Response $response): array
+    {
+        $this->checkAccess();
+        return $this->getService()->listSources();
+    }
+
+    public function getActionChunks(Request $request, Response $response): array
+    {
+        $this->checkAccess();
+        $sourceId = $request->getQueryParam('sourceId');
+
+        if (!$sourceId || !is_numeric($sourceId)) {
+            throw new BadRequest('sourceId (numeric) is required.');
+        }
+
+        return $this->getService()->sourceChunks((int) $sourceId);
+    }
+
+    public function postActionAsk(Request $request, Response $response): array
+    {
+        $this->checkAccess();
+        $data = $request->getParsedBody();
+
+        if (empty($data->message)) {
+            throw new BadRequest('message is required.');
+        }
+
+        return $this->getService()->ask(
+            trim($data->message),
+            $data->conversationId ?? null,
+            $data->conversationHistory ?? []
+        );
+    }
+}
@@ -0,0 +1,90 @@
+<?php
+
+namespace Espo\Modules\KnowledgeBase\EntryPoints;
+
+use Espo\Core\Api\Request;
+use Espo\Core\Api\Response;
+use Espo\Core\EntryPoint\EntryPoint;
+use Espo\Core\Exceptions\BadRequest;
+use Espo\Core\Exceptions\Error;
+use Espo\Core\Exceptions\Forbidden;
+use Espo\Core\Exceptions\NotFound;
+use Espo\Core\InjectableFactory;
+use Espo\Entities\User;
+use Espo\Modules\KnowledgeBase\Services\KnowledgeBaseService;
+
+/**
+ * Streams the original PDF of a KB source to an iframe in the browse tab.
+ * Access: `?entryPoint=KnowledgeBasePdf&sourceId=<id>`.
+ *
+ * We intentionally do NOT reuse EspoCRM's Attachment ACL model — KB sources
+ * are not Entities. The gate is the same rule as the Controller: no portal
+ * users. Anyone with a regular user account in the CRM can read the KB.
+ */
+class KnowledgeBasePdf implements EntryPoint
+{
+    public function __construct(
+        private InjectableFactory $injectableFactory,
+        private User $user,
+    ) {}
+
+    public function run(Request $request, Response $response): void
+    {
+        if ($this->user->isPortal()) {
+            throw new Forbidden('Portal users have no KB access.');
+        }
+
+        $sourceId = $request->getQueryParam('sourceId');
+        if (!$sourceId || !is_numeric($sourceId)) {
+            throw new BadRequest('sourceId (numeric) is required.');
+        }
+
+        $service = $this->injectableFactory->create(KnowledgeBaseService::class);
+
+        try {
+            $pdf = $service->getPdfBinary((int) $sourceId);
+        } catch (NotFound | Forbidden | BadRequest $e) {
+            throw $e;
+        } catch (\Throwable $e) {
+            throw new Error('Failed to fetch PDF: ' . $e->getMessage());
+        }
+
+        $body = (string) ($pdf['body'] ?? '');
+        $contentType = (string) ($pdf['contentType'] ?? 'application/pdf');
+        $filename = (string) ($pdf['filename'] ?? ('source-' . $sourceId . '.pdf'));
+
+        if ($body === '') {
+            throw new Error('Empty PDF body');
+        }
+
+        // Disposition: inline so the browser renders rather than downloads.
+        // Quote-escape the ASCII fallback; add RFC 5987 encoded UTF-8 for the
+        // real Hebrew filename.
+        $asciiFallback = 'source-' . (int) $sourceId . '.pdf';
+        $utf8Encoded = rawurlencode($filename);
+        $disposition = sprintf(
+            'inline; filename="%s"; filename*=UTF-8\'\'%s',
+            $asciiFallback,
+            $utf8Encoded,
+        );
+
+        // setBody() expects a PSR StreamInterface, not a raw string. Wrap
+        // the binary in an in-memory php://temp stream so the ResponseWrapper
+        // can stream it to the client without buffering twice.
+        $stream = fopen('php://temp', 'w+b');
+        fwrite($stream, $body);
+        rewind($stream);
+        $psrStream = \GuzzleHttp\Psr7\Utils::streamFor($stream);
+
+        $response
+            ->setHeader('Content-Type', $contentType)
+            ->setHeader('Content-Disposition', $disposition)
+            ->setHeader('Content-Length', (string) strlen($body))
+            ->setHeader('X-Content-Type-Options', 'nosniff')
+            ->setHeader('Cache-Control', 'private, max-age=300')
+            // Downloading via the attachment entry point uses a locked-down
+            // CSP; we do the same here to keep inline viewing safe.
+            ->setHeader('Content-Security-Policy', "default-src 'self'")
+            ->setBody($psrStream);
+    }
+}
@@ -0,0 +1,8 @@
+{
+    "scopeNames": {
+        "KnowledgeBase": "Knowledge Base"
+    },
+    "scopeNamesPlural": {
+        "KnowledgeBase": "Knowledge Base"
+    }
+}
@@ -0,0 +1,13 @@
+{
+    "labels": {
+        "Knowledge Base": "Knowledge Base",
+        "Search": "Search",
+        "Ask Shira": "Ask Shira",
+        "Browse": "Browse",
+        "Loading": "Loading…",
+        "No results": "No results"
+    },
+    "dashlets": {
+        "KbSearch": "KB Search"
+    }
+}
@@ -0,0 +1,8 @@
+{
+    "scopeNames": {
+        "KnowledgeBase": "בסיס ידע"
+    },
+    "scopeNamesPlural": {
+        "KnowledgeBase": "בסיס ידע"
+    }
+}
@@ -0,0 +1,13 @@
+{
+    "labels": {
+        "Knowledge Base": "בסיס ידע",
+        "Search": "חיפוש",
+        "Ask Shira": "שאל את שירה",
+        "Browse": "עיון",
+        "Loading": "טוען…",
+        "No results": "לא נמצאו תוצאות"
+    },
+    "dashlets": {
+        "KbSearch": "חיפוש בבסיס ידע"
+    }
+}
@@ -0,0 +1,8 @@
+{
+    "scopeNames": {
+        "KnowledgeBase": "בסיס ידע"
+    },
+    "scopeNamesPlural": {
+        "KnowledgeBase": "בסיס ידע"
+    }
+}
@@ -0,0 +1,13 @@
+{
+    "labels": {
+        "Knowledge Base": "בסיס ידע",
+        "Search": "חיפוש",
+        "Ask Shira": "שאל את שירה",
+        "Browse": "עיון",
+        "Loading": "טוען…",
+        "No results": "לא נמצאו תוצאות"
+    },
+    "dashlets": {
+        "KbSearch": "חיפוש בבסיס ידע"
+    }
+}
@@ -0,0 +1,6 @@
+{
+    "KbSearch": {
+        "view": "modules/knowledge-base/views/dashlets/kb-search",
+        "aclScope": "KnowledgeBase"
+    }
+}
@@ -0,0 +1,5 @@
+{
+    "KnowledgeBasePdf": {
+        "className": "Espo\\Modules\\KnowledgeBase\\EntryPoints\\KnowledgeBasePdf"
+    }
+}
@@ -0,0 +1,4 @@
+{
+    "controller": "modules/knowledge-base/controllers/kb",
+    "iconClass": "fas fa-book"
+}
@@ -0,0 +1,4 @@
+{
+    "tab": true,
+    "module": "KnowledgeBase"
+}
@@ -0,0 +1,34 @@
+[
+    {
+        "route": "/KnowledgeBase/action/search",
+        "method": "post",
+        "params": {
+            "controller": "KnowledgeBase",
+            "action": "search"
+        }
+    },
+    {
+        "route": "/KnowledgeBase/action/sources",
+        "method": "get",
+        "params": {
+            "controller": "KnowledgeBase",
+            "action": "sources"
+        }
+    },
+    {
+        "route": "/KnowledgeBase/action/chunks",
+        "method": "get",
+        "params": {
+            "controller": "KnowledgeBase",
+            "action": "chunks"
+        }
+    },
+    {
+        "route": "/KnowledgeBase/action/ask",
+        "method": "post",
+        "params": {
+            "controller": "KnowledgeBase",
+            "action": "ask"
+        }
+    }
+]
@@ -0,0 +1,251 @@
+<?php
+
+namespace Espo\Modules\KnowledgeBase\Services;
+
+use Espo\Core\Exceptions\BadRequest;
+use Espo\Core\Exceptions\Error;
+use Espo\Core\Exceptions\NotFound;
+use Espo\Core\Utils\Config;
+use Espo\Core\Utils\Log;
+use Espo\ORM\EntityManager;
+
+/**
+ * Thin HTTP client over shira-hermes `/kb/*` endpoints. Reuses the
+ * SmartAssistant integration record for connection details: the same
+ * shira-hermes instance hosts both the chat webhook and the KB API, so
+ * deriving the KB base URL from SmartAssistant's webhookUrl keeps
+ * everything in one place and avoids a second admin form.
+ */
+class KnowledgeBaseService
+{
+    private EntityManager $entityManager;
+    private Config $config;
+    private Log $log;
+
+    public function __construct(EntityManager $entityManager, Config $config, Log $log)
+    {
+        $this->entityManager = $entityManager;
+        $this->config = $config;
+        $this->log = $log;
+    }
+
+    public function search(string $query, string $kind, int $topK): array
+    {
+        $topK = max(1, min(15, $topK));
+        if (!in_array($kind, ['any', 'law', 'regulation', 'circular'], true)) {
+            $kind = 'any';
+        }
+
+        return $this->post('/kb/search', [
+            'query' => $query,
+            'kind' => $kind,
+            'top_k' => $topK,
+        ]);
+    }
+
+    public function listSources(): array
+    {
+        return $this->get('/kb/sources');
+    }
+
+    public function sourceChunks(int $sourceId): array
+    {
+        return $this->get('/kb/source/' . $sourceId . '/chunks');
+    }
+
+    public function ask(string $message, ?string $conversationId, array $history): array
+    {
+        return $this->post('/kb/ask', [
+            'message' => $message,
+            'conversation_id' => $conversationId,
+            'conversation_history' => $history,
+        ]);
+    }
+
+    /**
+     * Fetch the original PDF bytes of a source from shira-hermes.
+     *
+     * @return array{body:string, contentType:string, filename:string}
+     * @throws NotFound when the source doesn't exist in the KB
+     * @throws BadRequest when the source has no PDF (stored as plain text)
+     * @throws Error on transport or upstream 5xx errors
+     */
+    public function getPdfBinary(int $sourceId): array
+    {
+        $resp = $this->requestBinary('GET', '/kb/source/' . $sourceId . '/pdf');
+
+        $status = $resp['status'];
+        if ($status === 404) {
+            throw new NotFound('KB source not found.');
+        }
+        if ($status === 409) {
+            throw new BadRequest('This source has no PDF original.');
+        }
+        if ($status < 200 || $status >= 300) {
+            $this->log->error("KnowledgeBase: PDF HTTP {$status}");
+            throw new Error("Knowledge Base returned HTTP {$status}");
+        }
+
+        // Parse the UTF-8 filename from Content-Disposition if present, fall
+        // back to a generic name. The EntryPoint will re-emit the same shape.
+        $filename = 'source-' . $sourceId . '.pdf';
+        if (preg_match("/filename\\*=UTF-8''([^;\\s]+)/i", $resp['disposition'] ?? '', $m)) {
+            $filename = rawurldecode($m[1]);
+        }
+
+        return [
+            'body' => $resp['body'],
+            'contentType' => $resp['contentType'] ?: 'application/pdf',
+            'filename' => $filename,
+        ];
+    }
+
+    // ---- HTTP helpers ----------------------------------------------------
+
+    private function getBaseUrl(): string
+    {
+        $integration = $this->entityManager->getEntityById('Integration', 'SmartAssistant');
+        if (!$integration || !$integration->get('enabled')) {
+            throw new Error('SmartAssistant integration is not enabled; cannot reach shira-hermes KB.');
+        }
+        $data = $integration->get('data') ?? (object) [];
+        $webhookUrl = $data->webhookUrl ?? null;
+        if (!$webhookUrl) {
+            throw new Error('SmartAssistant webhook URL is not configured.');
+        }
+        // The webhook URL typically looks like: https://shira.dev.marcus-law.co.il/api/smart-assistant/chat
+        // We want the origin only — strip any path and trailing slash.
+        $parts = parse_url($webhookUrl);
+        if (!$parts || empty($parts['scheme']) || empty($parts['host'])) {
+            throw new Error('Webhook URL is not parseable.');
+        }
+        $origin = $parts['scheme'] . '://' . $parts['host'];
+        if (!empty($parts['port'])) {
+            $origin .= ':' . $parts['port'];
+        }
+        return $origin;
+    }
+
+    private function getApiKey(): ?string
+    {
+        $integration = $this->entityManager->getEntityById('Integration', 'SmartAssistant');
+        if (!$integration || !$integration->get('enabled')) {
+            return null;
+        }
+        $data = $integration->get('data') ?? (object) [];
+        return $data->apiKey ?? null;
+    }
+
+    private function get(string $path): array
+    {
+        return $this->request('GET', $path, null);
+    }
+
+    private function post(string $path, array $payload): array
+    {
+        return $this->request('POST', $path, $payload);
+    }
+
+    /**
+     * Variant of request() for binary responses (PDFs). No json_decode; splits
+     * the response into headers + body so callers can see Content-Type and
+     * Content-Disposition.
+     *
+     * @return array{body:string, contentType:string, disposition:string, status:int}
+     */
+    private function requestBinary(string $method, string $path): array
+    {
+        $url = $this->getBaseUrl() . $path;
+        $apiKey = $this->getApiKey();
+
+        $headers = ['Accept: application/pdf,*/*'];
+        if ($apiKey) {
+            $headers[] = 'X-Api-Key: ' . $apiKey;
+        }
+
+        $ch = curl_init($url);
+        curl_setopt_array($ch, [
+            CURLOPT_CUSTOMREQUEST => $method,
+            CURLOPT_HTTPHEADER => $headers,
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_HEADER => true,
+            CURLOPT_TIMEOUT => 180,
+            CURLOPT_CONNECTTIMEOUT => 10,
+        ]);
+
+        $raw = curl_exec($ch);
+        $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        $headerSize = (int) curl_getinfo($ch, CURLINFO_HEADER_SIZE);
+        $error = curl_error($ch);
+        curl_close($ch);
+
+        if ($error) {
+            $this->log->error("KnowledgeBase: HTTP error calling {$url}: {$error}");
+            throw new Error("Failed to reach Knowledge Base: {$error}");
+        }
+
+        $rawHeaders = substr($raw, 0, $headerSize) ?: '';
+        $body = substr($raw, $headerSize) ?: '';
+
+        $contentType = '';
+        $disposition = '';
+        foreach (explode("\r\n", $rawHeaders) as $line) {
+            if (stripos($line, 'Content-Type:') === 0) {
+                $contentType = trim(substr($line, strlen('Content-Type:')));
+            } elseif (stripos($line, 'Content-Disposition:') === 0) {
+                $disposition = trim(substr($line, strlen('Content-Disposition:')));
+            }
+        }
+
+        return [
+            'body' => $body,
+            'contentType' => $contentType,
+            'disposition' => $disposition,
+            'status' => $httpCode,
+        ];
+    }
+
+    private function request(string $method, string $path, ?array $payload): array
+    {
+        $url = $this->getBaseUrl() . $path;
+        $apiKey = $this->getApiKey();
+
+        $headers = ['Accept: application/json'];
+        if ($apiKey) {
+            $headers[] = 'X-Api-Key: ' . $apiKey;
+        }
+
+        $ch = curl_init($url);
+        $opts = [
+            CURLOPT_CUSTOMREQUEST => $method,
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_TIMEOUT => 180,
+            CURLOPT_CONNECTTIMEOUT => 10,
+        ];
+        if ($payload !== null) {
+            $headers[] = 'Content-Type: application/json';
+            $opts[CURLOPT_POSTFIELDS] = json_encode($payload);
+        }
+        $opts[CURLOPT_HTTPHEADER] = $headers;
+        curl_setopt_array($ch, $opts);
+
+        $body = curl_exec($ch);
+        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        $error = curl_error($ch);
+        curl_close($ch);
+
+        if ($error) {
+            $this->log->error("KnowledgeBase: HTTP error calling {$url}: {$error}");
+            throw new Error("Failed to reach Knowledge Base: {$error}");
+        }
+        if ($httpCode < 200 || $httpCode >= 300) {
+            $this->log->error("KnowledgeBase: HTTP {$httpCode} from {$url}: {$body}");
+            throw new Error("Knowledge Base returned HTTP {$httpCode}");
+        }
+        $decoded = json_decode($body, true);
+        if (!is_array($decoded)) {
+            throw new Error('Invalid JSON from Knowledge Base.');
+        }
+        return $decoded;
+    }
+}