diff --git a/.taskmaster/tasks/tasks.json b/.taskmaster/tasks/tasks.json index db3ab8b..47260b3 100644 --- a/.taskmaster/tasks/tasks.json +++ b/.taskmaster/tasks/tasks.json @@ -24,16 +24,28 @@ "priority": "high", "subtasks": [], "updatedAt": "2026-04-21T00:00:00.000Z" + }, + { + "id": "3", + "title": "Add תקנות הביטוח הלאומי (~100 regulations) from Wikisource to KB", + "description": "Extend scripts/fetch_national_insurance_law.py to bulk-fetch ~100 regulation pages from Hebrew Wikisource via MediaWiki API. Output text + sidecar JSON pairs to /tmp/regulations/. Upload to MinIO inbox/regulation/ for the existing scan-inbox cron to embed via Voyage. Estimated 30-40MB Postgres growth, ~$0.36 embedding cost. Sample 3 first (small/medium/large), verify chunker output, then bulk.", + "details": "", + "testStrategy": "", + "status": "done", + "dependencies": [], + "priority": "high", + "subtasks": [], + "updatedAt": "2026-04-25T20:39:11.905Z" } ], "metadata": { "version": "1.0.0", - "lastModified": "2026-04-21T00:00:00.000Z", - "taskCount": 2, - "completedCount": 0, + "lastModified": "2026-04-25T20:39:11.905Z", + "taskCount": 3, + "completedCount": 1, "tags": [ "master" ] } } -} +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c150736..9898afa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "shira-hermes" -version = "0.1.0" +version = "0.2.0" description = "Shira AI Assistant — Hermes Agent backend for EspoCRM SmartAssistant" requires-python = ">=3.11" dependencies = [ diff --git a/scripts/fetch_national_insurance_law.py b/scripts/fetch_national_insurance_law.py index 465cd38..4c627af 100755 --- a/scripts/fetch_national_insurance_law.py +++ b/scripts/fetch_national_insurance_law.py @@ -1,17 +1,28 @@ #!/usr/bin/env python3 -"""Fetch חוק הביטוח הלאומי from Hebrew Wikisource and convert to plain text -with the headers the shira-hermes chunker expects (חלק / פרק / סימן / סעיף).""" +"""Fetch חוק הביטוח הלאומי and its תקנות from Hebrew Wikisource and convert to +plain text with the headers the shira-hermes chunker expects (חלק / פרק / +סימן / סעיף). + +Modes: + (default) fetch the parent law (חוק הביטוח הלאומי) → /tmp/national_insurance_law.txt + --regulations fetch all תקנות הביטוח הלאומי canonical pages (~92, redirects skipped) + → /tmp/regulations/.txt + .txt.meta.json +""" from __future__ import annotations +import argparse import json +import os import re import sys +import time import urllib.request import urllib.parse WIKI_PAGE = "חוק_הביטוח_הלאומי" API = "https://he.wikisource.org/w/api.php" +REGULATIONS_PREFIX = "תקנות הביטוח הלאומי" def fetch_wikitext(page: str) -> str: @@ -217,8 +228,111 @@ def _cleanup(text: str) -> str: return text.strip() +def fetch_regulation_titles() -> list[str]: + """Query MediaWiki API for canonical (non-redirect) תקנות הביטוח הלאומי pages. + + Wikisource keeps duplicate-spelling pages (e.g. קיצבה/קצבה) as redirects to + the modern canonical title. Filtering apfilterredir=nonredirects gives us + the deduped set without having to maintain a hand-curated list. + """ + titles: list[str] = [] + apcontinue: str | None = None + while True: + params = { + "action": "query", + "list": "allpages", + "apprefix": REGULATIONS_PREFIX, + "apnamespace": "0", + "apfilterredir": "nonredirects", + "aplimit": "500", + "format": "json", + } + if apcontinue: + params["apcontinue"] = apcontinue + url = API + "?" + urllib.parse.urlencode(params) + req = urllib.request.Request(url, headers={"User-Agent": "shira-kb/1.0"}) + with urllib.request.urlopen(req) as r: + data = json.load(r) + titles.extend(p["title"] for p in data["query"]["allpages"]) + cont = data.get("continue", {}).get("apcontinue") + if not cont: + break + apcontinue = cont + return titles + + +_SLUG_DROP = re.compile(r"[()\[\]{}'\"־–—]") # parens, brackets, quotes, hyphens +_SLUG_WS = re.compile(r"[\s\-]+") + + +def _slug(title: str) -> str: + """Convert a Hebrew title into a safe filename — Hebrew letters preserved. + + Strips parens/brackets/quotes (problematic in shells and S3 console URLs), + collapses whitespace and hyphens to a single hyphen. + """ + s = _SLUG_DROP.sub(" ", title) + s = _SLUG_WS.sub("-", s.strip()) + return s + + +def fetch_one_regulation(title: str, out_dir: str) -> dict: + """Fetch a single regulation, expand templates, clean, write text + sidecar.""" + wt = fetch_wikitext(title) + expanded = _expand_templates(wt) + clean = _cleanup(expanded) + slug = _slug(title) + txt_path = os.path.join(out_dir, f"{slug}.txt") + meta_path = txt_path + ".meta.json" + with open(txt_path, "w", encoding="utf-8") as f: + f.write(clean) + sidecar = { + "title": title, + "identifier": title, + "source_url": f"https://he.wikisource.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}", + } + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(sidecar, f, ensure_ascii=False, indent=2) + return {"title": title, "slug": slug, "raw_chars": len(wt), "clean_chars": len(clean)} + + +def fetch_all_regulations(out_dir: str) -> int: + """Fetch every canonical תקנות הביטוח הלאומי page into out_dir.""" + os.makedirs(out_dir, exist_ok=True) + titles = fetch_regulation_titles() + print(f"[regulations] {len(titles)} canonical pages", file=sys.stderr) + total_clean = 0 + for i, title in enumerate(titles, 1): + try: + r = fetch_one_regulation(title, out_dir) + total_clean += r["clean_chars"] + print( + f"[{i:3d}/{len(titles)}] {r['slug']:.<70s} " + f"raw={r['raw_chars']:>7,} clean={r['clean_chars']:>7,}", + file=sys.stderr, + ) + except Exception as e: + print(f"[{i:3d}/{len(titles)}] FAILED {title}: {e}", file=sys.stderr) + # Be polite to Wikisource — 50ms between requests = ~20 req/s, well under any limit. + time.sleep(0.05) + print(f"[done] total clean text: {total_clean:,} chars across {len(titles)} regulations", + file=sys.stderr) + return 0 + + def main() -> int: - out_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/national_insurance_law.txt" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--regulations", action="store_true", + help="fetch all תקנות (default: fetch the parent law only)") + parser.add_argument("out", nargs="?", default=None, + help="output path (default: /tmp/national_insurance_law.txt for law, /tmp/regulations/ for --regulations)") + args = parser.parse_args() + + if args.regulations: + out_dir = args.out or "/tmp/regulations" + return fetch_all_regulations(out_dir) + + out_path = args.out or "/tmp/national_insurance_law.txt" wt = fetch_wikitext(WIKI_PAGE) print(f"[fetch] raw wikitext: {len(wt):,} chars", file=sys.stderr) expanded = _expand_templates(wt)