feat(kb): Phase 1 — Israeli National Insurance knowledge base
Adds a searchable KB for חוק הביטוח הלאומי, תקנות הביטוח הלאומי, and חוזרי הביטוח הלאומי. Hybrid search (pgvector cosine + tsvector/trgm) fused with Reciprocal Rank Fusion, exposed to Shira as the `search_insurance_kb` tool. - api/services/kb/: asyncpg pool, Voyage voyage-multilingual-2 client, per-kind chunker (statute-by-section / circular header-aware), ingest pipeline with supersession (old source → superseded_by), hybrid search. - api/routes/admin_kb.py: POST /admin/kb/ingest (multipart), GET /admin/kb/stats. - mcp_server/tools/legal_kb_tools.py: `search_insurance_kb` registered under the `legal` toolset. - pyproject.toml: +asyncpg, +python-docx, +boto3, +python-multipart. Infra (external): pgvector/pgvector:pg16 on the shared PG, insurance_kb DB with hnsw + gin indexes, MinIO bucket insurance-kb, KB_DATABASE_URL in Infisical /espocrm, VOYAGE_API_KEY/VOYAGE_MODEL/KB_DATABASE_URL on the shira-hermes Coolify app. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
"""Voyage embeddings client — voyage-multilingual-2, 1024-dim."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger("shira.kb.voyage")
|
||||
|
||||
_VOYAGE_URL = "https://api.voyageai.com/v1/embeddings"
|
||||
_MAX_BATCH = 128
|
||||
|
||||
|
||||
class VoyageError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
async def embed(
|
||||
texts: list[str],
|
||||
input_type: Literal["document", "query"] = "document",
|
||||
model: str | None = None,
|
||||
) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
|
||||
api_key = os.environ.get("VOYAGE_API_KEY")
|
||||
if not api_key:
|
||||
raise VoyageError("VOYAGE_API_KEY is not set")
|
||||
|
||||
model = model or os.environ.get("VOYAGE_MODEL", "voyage-multilingual-2")
|
||||
|
||||
out: list[list[float]] = []
|
||||
async with httpx.AsyncClient(timeout=60) as http:
|
||||
for start in range(0, len(texts), _MAX_BATCH):
|
||||
batch = texts[start : start + _MAX_BATCH]
|
||||
resp = await http.post(
|
||||
_VOYAGE_URL,
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
json={"model": model, "input": batch, "input_type": input_type},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise VoyageError(f"Voyage API {resp.status_code}: {resp.text[:200]}")
|
||||
data = resp.json()
|
||||
for item in sorted(data["data"], key=lambda d: d["index"]):
|
||||
out.append(item["embedding"])
|
||||
logger.info(
|
||||
"[kb.voyage] embedded batch=%d/%d type=%s",
|
||||
min(start + _MAX_BATCH, len(texts)),
|
||||
len(texts),
|
||||
input_type,
|
||||
)
|
||||
return out
|
||||
Reference in New Issue
Block a user