feat(kb): bulk-fetch תקנות הביטוח הלאומי from Wikisource

Extends scripts/fetch_national_insurance_law.py with a --regulations mode
that pulls all canonical (non-redirect) תקנות הביטוח הלאומי pages from the
MediaWiki API and writes <slug>.txt + <slug>.txt.meta.json pairs ready for
the existing scan-inbox cron.

Used to seed the live insurance KB with 92 regulations (~1,500 chunks,
~22MB Postgres) on dev. Wikisource's redirect graph dedups spelling
variants (קיצבה/קצבה, etc.) for free.

Bumps version to 0.2.0.

Refs Task Master #3

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.taskmaster/tasks/tasks.json

@@ -24,13 +24,25 @@
         "priority": "high",
         "subtasks": [],
         "updatedAt": "2026-04-21T00:00:00.000Z"
+      },
+      {
+        "id": "3",
+        "title": "Add תקנות הביטוח הלאומי (~100 regulations) from Wikisource to KB",
+        "description": "Extend scripts/fetch_national_insurance_law.py to bulk-fetch ~100 regulation pages from Hebrew Wikisource via MediaWiki API. Output text + sidecar JSON pairs to /tmp/regulations/. Upload to MinIO inbox/regulation/ for the existing scan-inbox cron to embed via Voyage. Estimated 30-40MB Postgres growth, ~$0.36 embedding cost. Sample 3 first (small/medium/large), verify chunker output, then bulk.",
+        "details": "",
+        "testStrategy": "",
+        "status": "done",
+        "dependencies": [],
+        "priority": "high",
+        "subtasks": [],
+        "updatedAt": "2026-04-25T20:39:11.905Z"
       }
     ],
     "metadata": {
       "version": "1.0.0",
-      "lastModified": "2026-04-21T00:00:00.000Z",
+      "lastModified": "2026-04-25T20:39:11.905Z",
-      "taskCount": 2,
+      "taskCount": 3,
-      "completedCount": 0,
+      "completedCount": 1,
       "tags": [
         "master"
       ]

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "shira-hermes"
-version = "0.1.0"
+version = "0.2.0"
 description = "Shira AI Assistant — Hermes Agent backend for EspoCRM SmartAssistant"
 requires-python = ">=3.11"
 dependencies = [

scripts/fetch_national_insurance_law.py

@@ -1,17 +1,28 @@
 #!/usr/bin/env python3
-"""Fetch חוק הביטוח הלאומי from Hebrew Wikisource and convert to plain text
+"""Fetch חוק הביטוח הלאומי and its תקנות from Hebrew Wikisource and convert to
-with the headers the shira-hermes chunker expects (חלק / פרק / סימן / סעיף)."""
+plain text with the headers the shira-hermes chunker expects (חלק / פרק /
+סימן / סעיף).
+
+Modes:
+  (default)         fetch the parent law (חוק הביטוח הלאומי) → /tmp/national_insurance_law.txt
+  --regulations     fetch all תקנות הביטוח הלאומי canonical pages (~92, redirects skipped)
+                    → /tmp/regulations/<slug>.txt + <slug>.txt.meta.json
+"""
 
 from __future__ import annotations
 
+import argparse
 import json
+import os
 import re
 import sys
+import time
 import urllib.request
 import urllib.parse
 
 WIKI_PAGE = "חוק_הביטוח_הלאומי"
 API = "https://he.wikisource.org/w/api.php"
+REGULATIONS_PREFIX = "תקנות הביטוח הלאומי"
 
 
 def fetch_wikitext(page: str) -> str:
@@ -217,8 +228,111 @@ def _cleanup(text: str) -> str:
     return text.strip()
 
 
+def fetch_regulation_titles() -> list[str]:
+    """Query MediaWiki API for canonical (non-redirect) תקנות הביטוח הלאומי pages.
+
+    Wikisource keeps duplicate-spelling pages (e.g. קיצבה/קצבה) as redirects to
+    the modern canonical title. Filtering apfilterredir=nonredirects gives us
+    the deduped set without having to maintain a hand-curated list.
+    """
+    titles: list[str] = []
+    apcontinue: str | None = None
+    while True:
+        params = {
+            "action": "query",
+            "list": "allpages",
+            "apprefix": REGULATIONS_PREFIX,
+            "apnamespace": "0",
+            "apfilterredir": "nonredirects",
+            "aplimit": "500",
+            "format": "json",
+        }
+        if apcontinue:
+            params["apcontinue"] = apcontinue
+        url = API + "?" + urllib.parse.urlencode(params)
+        req = urllib.request.Request(url, headers={"User-Agent": "shira-kb/1.0"})
+        with urllib.request.urlopen(req) as r:
+            data = json.load(r)
+        titles.extend(p["title"] for p in data["query"]["allpages"])
+        cont = data.get("continue", {}).get("apcontinue")
+        if not cont:
+            break
+        apcontinue = cont
+    return titles
+
+
+_SLUG_DROP = re.compile(r"[()\[\]{}'\"־–—]")  # parens, brackets, quotes, hyphens
+_SLUG_WS = re.compile(r"[\s\-]+")
+
+
+def _slug(title: str) -> str:
+    """Convert a Hebrew title into a safe filename — Hebrew letters preserved.
+
+    Strips parens/brackets/quotes (problematic in shells and S3 console URLs),
+    collapses whitespace and hyphens to a single hyphen.
+    """
+    s = _SLUG_DROP.sub(" ", title)
+    s = _SLUG_WS.sub("-", s.strip())
+    return s
+
+
+def fetch_one_regulation(title: str, out_dir: str) -> dict:
+    """Fetch a single regulation, expand templates, clean, write text + sidecar."""
+    wt = fetch_wikitext(title)
+    expanded = _expand_templates(wt)
+    clean = _cleanup(expanded)
+    slug = _slug(title)
+    txt_path = os.path.join(out_dir, f"{slug}.txt")
+    meta_path = txt_path + ".meta.json"
+    with open(txt_path, "w", encoding="utf-8") as f:
+        f.write(clean)
+    sidecar = {
+        "title": title,
+        "identifier": title,
+        "source_url": f"https://he.wikisource.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}",
+    }
+    with open(meta_path, "w", encoding="utf-8") as f:
+        json.dump(sidecar, f, ensure_ascii=False, indent=2)
+    return {"title": title, "slug": slug, "raw_chars": len(wt), "clean_chars": len(clean)}
+
+
+def fetch_all_regulations(out_dir: str) -> int:
+    """Fetch every canonical תקנות הביטוח הלאומי page into out_dir."""
+    os.makedirs(out_dir, exist_ok=True)
+    titles = fetch_regulation_titles()
+    print(f"[regulations] {len(titles)} canonical pages", file=sys.stderr)
+    total_clean = 0
+    for i, title in enumerate(titles, 1):
+        try:
+            r = fetch_one_regulation(title, out_dir)
+            total_clean += r["clean_chars"]
+            print(
+                f"[{i:3d}/{len(titles)}] {r['slug']:.<70s} "
+                f"raw={r['raw_chars']:>7,}  clean={r['clean_chars']:>7,}",
+                file=sys.stderr,
+            )
+        except Exception as e:
+            print(f"[{i:3d}/{len(titles)}] FAILED {title}: {e}", file=sys.stderr)
+        # Be polite to Wikisource — 50ms between requests = ~20 req/s, well under any limit.
+        time.sleep(0.05)
+    print(f"[done] total clean text: {total_clean:,} chars across {len(titles)} regulations",
+          file=sys.stderr)
+    return 0
+
+
 def main() -> int:
-    out_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/national_insurance_law.txt"
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--regulations", action="store_true",
+                        help="fetch all תקנות (default: fetch the parent law only)")
+    parser.add_argument("out", nargs="?", default=None,
+                        help="output path (default: /tmp/national_insurance_law.txt for law, /tmp/regulations/ for --regulations)")
+    args = parser.parse_args()
+
+    if args.regulations:
+        out_dir = args.out or "/tmp/regulations"
+        return fetch_all_regulations(out_dir)
+
+    out_path = args.out or "/tmp/national_insurance_law.txt"
     wt = fetch_wikitext(WIKI_PAGE)
     print(f"[fetch] raw wikitext: {len(wt):,} chars", file=sys.stderr)
     expanded = _expand_templates(wt)