feat(kb): per-document metadata via sidecar JSON

Each file in the inbox can now be accompanied by a `<filename>.meta.json`
sidecar that supplies identifier, published_at, effective_at, and
source_url. The scanner applies filename-derived defaults first and lets
the sidecar override individual fields.

- list_inbox / list_failed skip `.meta.json` so sidecars aren't ingested
  as standalone documents.
- mark_processed, mark_failed, and requeue-failed move the sidecar
  alongside its parent (best-effort).
- mark_failed now sanitizes the error string to ASCII before writing it
  to S3 object metadata (HTTP header encoding).
- search SQL selects published_at/effective_at; the formatter shows the
  full heading_path plus publication date so citations are unambiguous.
- scripts/kb_sidecar_template.json documents the expected shape.

Refs Task Master #2

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

api/routes/admin_kb.py

@@ -80,13 +80,45 @@ def _derive_metadata(filename: str, kind: str) -> dict:
         if stem.lower().endswith(ext):
             stem = stem[: -len(ext)]
             break
-    # Title is just the stem; identifier left empty unless obvious.
-    return {"title": stem, "identifier": None}
+    return {
+        "title": stem,
+        "identifier": None,
+        "published_at": None,
+        "effective_at": None,
+        "source_url": None,
+    }
+
+
+def _sidecar_metadata(item_key: str) -> dict:
+    """Fetch the optional `<key>.meta.json` sidecar and return parsed fields.
+
+    The sidecar is expected to be a JSON object with any of:
+      {title, identifier, published_at, effective_at, source_url}
+
+    Missing or unreadable sidecars return {}.
+    """
+    import json
+    sidecar_key = item_key + ".meta.json"
+    try:
+        data = kb_s3.fetch(sidecar_key)
+    except Exception:
+        return {}
+    try:
+        parsed = json.loads(data.decode("utf-8"))
+    except Exception as e:
+        logger.warning("[admin.kb] bad sidecar %s: %s", sidecar_key, e)
+        return {}
+    # Accept only whitelisted keys — ignore typos / extra fields.
+    allowed = {"title", "identifier", "published_at", "effective_at", "source_url"}
+    return {k: v for k, v in parsed.items() if k in allowed and v}
 
 
 @router.post("/requeue-failed")
 async def requeue_failed(request: Request):
-    """Move everything under failed/<kind>/ back to inbox/<kind>/ for retry."""
+    """Move everything under failed/<kind>/ back to inbox/<kind>/ for retry.
+
+    The sidecar `<name>.meta.json` (if present) is moved alongside.
+    """
     _verify_admin(request)
     items = kb_s3.list_failed()
     moved = []
@@ -95,6 +127,11 @@ async def requeue_failed(request: Request):
         dst = f"inbox/{item['kind']}/{item['filename']}"
         try:
             kb_s3.move(src, dst)
+            # Best-effort sidecar move.
+            try:
+                kb_s3.move(src + ".meta.json", dst + ".meta.json")
+            except Exception:
+                pass
             moved.append({"from": src, "to": dst})
         except Exception as e:
             moved.append({"from": src, "error": str(e)})
@@ -110,15 +147,20 @@ async def scan_inbox(request: Request):
         src_key = item["key"]
         kind = item["kind"]
         filename = item["filename"]
+        # Merge: filename-derived defaults < sidecar overrides.
         meta = _derive_metadata(filename, kind)
+        meta.update(_sidecar_metadata(src_key))
         try:
             data = kb_s3.fetch(src_key)
             result = await kb_ingest.ingest_source(
                 kind=kind,
                 title=meta["title"],
-                identifier=meta["identifier"],
+                identifier=meta.get("identifier"),
                 content=data,
                 filename=filename,
+                published_at=meta.get("published_at"),
+                effective_at=meta.get("effective_at"),
+                source_url=meta.get("source_url"),
                 original_path=f"s3://{kb_s3._bucket()}/processed/{kind}/{filename}",
             )
             dst = kb_s3.mark_processed(src_key, filename, kind)