diff --git a/scripts/fetch_national_insurance_law.py b/scripts/fetch_national_insurance_law.py new file mode 100755 index 0000000..465cd38 --- /dev/null +++ b/scripts/fetch_national_insurance_law.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Fetch חוק הביטוח הלאומי from Hebrew Wikisource and convert to plain text +with the headers the shira-hermes chunker expects (חלק / פרק / סימן / סעיף).""" + +from __future__ import annotations + +import json +import re +import sys +import urllib.request +import urllib.parse + +WIKI_PAGE = "חוק_הביטוח_הלאומי" +API = "https://he.wikisource.org/w/api.php" + + +def fetch_wikitext(page: str) -> str: + url = f"{API}?action=parse&page={urllib.parse.quote(page)}&prop=wikitext&format=json" + req = urllib.request.Request(url, headers={"User-Agent": "shira-kb/1.0"}) + with urllib.request.urlopen(req) as r: + data = json.load(r) + return data["parse"]["wikitext"]["*"] + + +# ---- template handlers ---------------------------------------------------- + +# {{ח:קטע2|סימולא|שם החלק}} → "חלק " +# Wikisource numbers parts/chapters in their own way; sometimes the number is +# embedded in the title. We emit the raw text of the second param and let the +# chunker pick up the prefix from there. +def _section_header(prefix: str, args: list[str]) -> str: + args = _positional_args(args) + if not args: + return "" + # The title is usually the last positional arg; for קטע templates the + # title already contains its own prefix ("פרק א׳: ...") so we don't + # add our own. Otherwise prepend the given prefix. + title = (args[-1] if args[-1].strip() else (args[0] if args else "")).strip() + if not title: + return "" + if prefix: + return f"\n\n{prefix} {title}\n\n" + return f"\n\n{title}\n\n" + + +# Templates that produce a statute heading line. +# Note: {{ח:קטע2}} / {{ח:קטע3}} don't directly encode the level name — +# the actual level is embedded in the *title* argument (e.g. "פרק א׳: ..."). +# So we emit the title verbatim and let the chunker's existing regexes +# pick it up (which they do, for פרק / סימן prefixes). +_HEADER_TEMPLATES = { + "ח:קטע2": "", + "ח:קטע3": "", + "ח:קטע4": "", +} + + +def _positional_args(args: list[str]) -> list[str]: + """Filter out named arguments (e.g. 'אחר=[פרק א׳]') from a template arg list.""" + return [a for a in args if "=" not in a.split("\n", 1)[0][:40]] + + +def _render_section(args: list[str]) -> str: + """{{ח:סעיף|number|title|anchor?}} → "סעיף <number>. <title>" """ + args = _positional_args(args) + if not args: + return "" + number = args[0].strip() + title = (args[1] if len(args) > 1 else "").strip() + if title: + return f"\n\nסעיף {number}. {title}\n\n" + return f"\n\nסעיף {number}.\n\n" + + +def _render_subsection(args: list[str]) -> str: + """{{ח:ת|א|text}} → "(א) text" on its own line""" + args = _positional_args(args) + if not args: + return "" + letter = args[0].strip() + body = (args[1] if len(args) > 1 else "").strip() + return f"\n({letter}) {body}" + + +def _render_linkish(args: list[str]) -> str: + """{{ח:פנימי|anchor|label}} / {{ח:חיצוני|url|label}} / {{ח:תיבה|...}} → + just the label text, dropping the link itself.""" + if not args: + return "" + # label is typically the second arg; if only one, return that. + return args[1].strip() if len(args) > 1 else args[0].strip() + + +def _drop(args: list[str]) -> str: + return "" + + +# Map template name → handler. Unknown templates are silently stripped. +_HANDLERS = { + "ח:סעיף": _render_section, + "ח:סעיף*": _render_section, + "ח:ת": _render_subsection, + "ח:תת": _render_subsection, + "ח:תתת": _render_subsection, + "ח:תתתת": _render_subsection, + "ח:תתתתת": _render_subsection, + "ח:תתתתתת": _render_subsection, + "ח:פנימי": _render_linkish, + "ח:חיצוני": _render_linkish, + "ח:תיבה": _render_linkish, + # No body — drop silently + "ח:התחלה": _drop, + "ח:סוף": _drop, + "ח:כותרת": _drop, + "ח:מאגר": _drop, + "ח:פתיח-התחלה": _drop, + "ח:מפריד": _drop, + "ח:סוגר": _drop, + "ח:מבוא": _drop, + "ח:חתימות": _drop, + "ח:הערה": _drop, # footnotes — not needed for RAG +} + + +def _parse_template_args(raw: str) -> list[str]: + """Split a template body by '|' at depth 0 (respecting nested templates).""" + args: list[str] = [] + buf: list[str] = [] + depth = 0 + i = 0 + while i < len(raw): + c = raw[i] + if c == "{" and i + 1 < len(raw) and raw[i + 1] == "{": + depth += 1 + buf.append("{{") + i += 2 + continue + if c == "}" and i + 1 < len(raw) and raw[i + 1] == "}": + depth -= 1 + buf.append("}}") + i += 2 + continue + if c == "[" and i + 1 < len(raw) and raw[i + 1] == "[": + depth += 1 + buf.append("[[") + i += 2 + continue + if c == "]" and i + 1 < len(raw) and raw[i + 1] == "]": + depth -= 1 + buf.append("]]") + i += 2 + continue + if c == "|" and depth == 0: + args.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(c) + i += 1 + args.append("".join(buf)) + return args + + +def _expand_templates(text: str) -> str: + """Recursively resolve {{ח:...}} templates, depth-first.""" + out: list[str] = [] + i = 0 + while i < len(text): + if text[i : i + 2] == "{{": + # find matching }} + depth = 1 + j = i + 2 + while j < len(text) and depth > 0: + if text[j : j + 2] == "{{": + depth += 1 + j += 2 + elif text[j : j + 2] == "}}": + depth -= 1 + j += 2 + else: + j += 1 + body = text[i + 2 : j - 2] # inside {{ ... }} + # recurse into body first + expanded_body = _expand_templates(body) + args = _parse_template_args(expanded_body) + name = args[0].strip() if args else "" + if name in _HEADER_TEMPLATES: + out.append(_section_header(_HEADER_TEMPLATES[name], args[1:])) + elif name in _HANDLERS: + out.append(_HANDLERS[name](args[1:])) + else: + # Unknown template — keep the last arg if it looks like text. + if len(args) > 1: + candidate = args[-1].strip() + if candidate and len(candidate) < 200: + out.append(candidate) + i = j + else: + out.append(text[i]) + i += 1 + return "".join(out) + + +def _cleanup(text: str) -> str: + # Drop wiki tables/images we don't care about. + text = re.sub(r"\[\[קובץ:[^\]]*\]\]", "", text) + text = re.sub(r"\[\[(?:[^\]|]*\|)?([^\]]*)\]\]", r"\1", text) # [[link|label]] → label + # Drop HTML-ish refs (footnotes, comments). + text = re.sub(r"<ref[^>]*>.*?</ref>", "", text, flags=re.DOTALL) + text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL) + text = re.sub(r"</?[a-zA-Z][^>]*>", "", text) + # Collapse 3+ newlines to 2; strip trailing spaces. + text = re.sub(r"[ \t]+\n", "\n", text) + text = re.sub(r"\n{3,}", "\n\n", text) + # Drop bold/italic wiki markup. + text = text.replace("'''", "").replace("''", "") + return text.strip() + + +def main() -> int: + out_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/national_insurance_law.txt" + wt = fetch_wikitext(WIKI_PAGE) + print(f"[fetch] raw wikitext: {len(wt):,} chars", file=sys.stderr) + expanded = _expand_templates(wt) + print(f"[expand] after templates: {len(expanded):,} chars", file=sys.stderr) + clean = _cleanup(expanded) + print(f"[cleanup] final: {len(clean):,} chars", file=sys.stderr) + with open(out_path, "w", encoding="utf-8") as f: + f.write(clean) + print(f"[write] {out_path}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main())