chaim
3d5bc38401
Each file in the inbox can now be accompanied by a `<filename>.meta.json` sidecar that supplies identifier, published_at, effective_at, and source_url. The scanner applies filename-derived defaults first and lets the sidecar override individual fields. - list_inbox / list_failed skip `.meta.json` so sidecars aren't ingested as standalone documents. - mark_processed, mark_failed, and requeue-failed move the sidecar alongside its parent (best-effort). - mark_failed now sanitizes the error string to ASCII before writing it to S3 object metadata (HTTP header encoding). - search SQL selects published_at/effective_at; the formatter shows the full heading_path plus publication date so citations are unambiguous. - scripts/kb_sidecar_template.json documents the expected shape. Refs Task Master #2 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
85 lines
2.3 KiB
Python
85 lines
2.3 KiB
Python
"""Hybrid search: vector + full-text, fused with Reciprocal Rank Fusion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Literal
|
|
|
|
from api.services.kb import voyage
|
|
from api.services.kb.db import get_pool
|
|
|
|
logger = logging.getLogger("shira.kb.search")
|
|
|
|
_RRF_K = 60 # standard RRF constant
|
|
_CANDIDATES_PER_SIDE = 30
|
|
|
|
|
|
async def search(
|
|
query: str,
|
|
kind: Literal["law", "regulation", "circular", "any"] = "any",
|
|
top_k: int = 8,
|
|
) -> list[dict]:
|
|
query = (query or "").strip()
|
|
if not query:
|
|
return []
|
|
|
|
[vec] = await voyage.embed([query], input_type="query")
|
|
|
|
kind_filter = ""
|
|
params: list = [vec, query, _CANDIDATES_PER_SIDE]
|
|
if kind != "any":
|
|
kind_filter = "AND s.kind = $4"
|
|
params.append(kind)
|
|
|
|
# Two CTEs — vector rank and lexical rank — fused via RRF.
|
|
sql = f"""
|
|
WITH vec AS (
|
|
SELECT c.id, ROW_NUMBER() OVER (ORDER BY c.embedding <=> $1::vector) AS rank
|
|
FROM kb_chunk c
|
|
JOIN kb_source s ON s.id = c.source_id
|
|
WHERE c.embedding IS NOT NULL
|
|
AND s.superseded_by IS NULL
|
|
{kind_filter}
|
|
ORDER BY c.embedding <=> $1::vector
|
|
LIMIT $3
|
|
),
|
|
lex AS (
|
|
SELECT c.id, ROW_NUMBER() OVER (ORDER BY ts_rank_cd(c.content_tsv, q) DESC) AS rank
|
|
FROM kb_chunk c
|
|
JOIN kb_source s ON s.id = c.source_id,
|
|
websearch_to_tsquery('simple', $2) AS q
|
|
WHERE c.content_tsv @@ q
|
|
AND s.superseded_by IS NULL
|
|
{kind_filter}
|
|
ORDER BY ts_rank_cd(c.content_tsv, q) DESC
|
|
LIMIT $3
|
|
),
|
|
fused AS (
|
|
SELECT id, SUM(score) AS score FROM (
|
|
SELECT id, 1.0 / ({_RRF_K} + rank) AS score FROM vec
|
|
UNION ALL
|
|
SELECT id, 1.0 / ({_RRF_K} + rank) AS score FROM lex
|
|
) u GROUP BY id
|
|
)
|
|
SELECT
|
|
s.kind, s.title, s.identifier, s.source_url,
|
|
s.published_at, s.effective_at,
|
|
c.heading_path, c.section_ref, c.content,
|
|
f.score
|
|
FROM fused f
|
|
JOIN kb_chunk c ON c.id = f.id
|
|
JOIN kb_source s ON s.id = c.source_id
|
|
ORDER BY f.score DESC
|
|
LIMIT {int(top_k)}
|
|
"""
|
|
|
|
pool = await get_pool()
|
|
async with pool.acquire() as conn:
|
|
rows = await conn.fetch(sql, *params)
|
|
|
|
logger.info(
|
|
"[kb.search] query=%r kind=%s hits=%d",
|
|
query[:80], kind, len(rows),
|
|
)
|
|
return [dict(r) for r in rows]
|