chaim
9edcb58c93
Adds a searchable KB for חוק הביטוח הלאומי, תקנות הביטוח הלאומי, and חוזרי הביטוח הלאומי. Hybrid search (pgvector cosine + tsvector/trgm) fused with Reciprocal Rank Fusion, exposed to Shira as the `search_insurance_kb` tool. - api/services/kb/: asyncpg pool, Voyage voyage-multilingual-2 client, per-kind chunker (statute-by-section / circular header-aware), ingest pipeline with supersession (old source → superseded_by), hybrid search. - api/routes/admin_kb.py: POST /admin/kb/ingest (multipart), GET /admin/kb/stats. - mcp_server/tools/legal_kb_tools.py: `search_insurance_kb` registered under the `legal` toolset. - pyproject.toml: +asyncpg, +python-docx, +boto3, +python-multipart. Infra (external): pgvector/pgvector:pg16 on the shared PG, insurance_kb DB with hnsw + gin indexes, MinIO bucket insurance-kb, KB_DATABASE_URL in Infisical /espocrm, VOYAGE_API_KEY/VOYAGE_MODEL/KB_DATABASE_URL on the shira-hermes Coolify app. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
"""Voyage embeddings client — voyage-multilingual-2, 1024-dim."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from typing import Literal
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger("shira.kb.voyage")
|
|
|
|
_VOYAGE_URL = "https://api.voyageai.com/v1/embeddings"
|
|
_MAX_BATCH = 128
|
|
|
|
|
|
class VoyageError(RuntimeError):
|
|
pass
|
|
|
|
|
|
async def embed(
|
|
texts: list[str],
|
|
input_type: Literal["document", "query"] = "document",
|
|
model: str | None = None,
|
|
) -> list[list[float]]:
|
|
if not texts:
|
|
return []
|
|
|
|
api_key = os.environ.get("VOYAGE_API_KEY")
|
|
if not api_key:
|
|
raise VoyageError("VOYAGE_API_KEY is not set")
|
|
|
|
model = model or os.environ.get("VOYAGE_MODEL", "voyage-multilingual-2")
|
|
|
|
out: list[list[float]] = []
|
|
async with httpx.AsyncClient(timeout=60) as http:
|
|
for start in range(0, len(texts), _MAX_BATCH):
|
|
batch = texts[start : start + _MAX_BATCH]
|
|
resp = await http.post(
|
|
_VOYAGE_URL,
|
|
headers={"Authorization": f"Bearer {api_key}"},
|
|
json={"model": model, "input": batch, "input_type": input_type},
|
|
)
|
|
if resp.status_code != 200:
|
|
raise VoyageError(f"Voyage API {resp.status_code}: {resp.text[:200]}")
|
|
data = resp.json()
|
|
for item in sorted(data["data"], key=lambda d: d["index"]):
|
|
out.append(item["embedding"])
|
|
logger.info(
|
|
"[kb.voyage] embedded batch=%d/%d type=%s",
|
|
min(start + _MAX_BATCH, len(texts)),
|
|
len(texts),
|
|
input_type,
|
|
)
|
|
return out
|