shira-hermes/scripts/migrate_legacy_memory_notes.py

"""Migrate legacy "🧠 category: content" Notes into CaseMemory rows.

History: until SmartAssistant 2.10.0 the Python `save_memory` tool created a
Note entity with `post` prefixed by "🧠 {category}:" instead of using the
proper CaseMemory entity. This left CaseMemory empty even though the system
prompt was reading from it — so case memory never accumulated.

This one-shot script reads every such Note in EspoCRM (across all cases),
parses out the category and content, creates a corresponding CaseMemory row
with source='assistant', and soft-deletes the original Note.

Usage (from inside a shira-hermes container that already has env vars set):
    python3 scripts/migrate_legacy_memory_notes.py [--dry-run]

Safe to re-run: matches `post LIKE "🧠 %"` only; soft-deletes after migration.
"""

from __future__ import annotations

import argparse
import os
import re
import sys
import time
from typing import Any

import httpx

VALID_CATEGORIES = {
    "key_facts", "strategy", "decisions",
    "contacts_notes", "timeline",
    "documents_notes", "billing_notes",
}

EMOJI_PREFIX_RE = re.compile(r"^🧠\s*([a-z_]+)\s*:\s*(.+)$", re.DOTALL)


def parse_note_post(post: str) -> tuple[str, str] | None:
    """Return (category, content) from a "🧠 category: content" string, or None."""
    if not post or not post.startswith("🧠"):
        return None
    m = EMOJI_PREFIX_RE.match(post.strip())
    if not m:
        return None
    category, content = m.group(1).strip(), m.group(2).strip()
    if category not in VALID_CATEGORIES:
        category = "key_facts"
    return category, content


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--dry-run", action="store_true", help="Don't write anything; just report what would happen.")
    args = parser.parse_args()

    base = os.environ.get("ESPOCRM_URL", "").rstrip("/")
    key = os.environ.get("ESPOCRM_API_KEY", "")
    if not base or not key:
        print("ESPOCRM_URL or ESPOCRM_API_KEY missing in env. Exiting.", file=sys.stderr)
        return 2

    headers = {"X-Api-Key": key, "Content-Type": "application/json"}

    # Page through notes that look like memory entries.
    offset = 0
    page_size = 100
    migrated = 0
    skipped = 0
    failed = 0

    while True:
        params = {
            "where[0][type]": "startsWith",
            "where[0][attribute]": "post",
            "where[0][value]": "🧠",
            "maxSize": page_size,
            "offset": offset,
            "select": "id,post,parentId,parentType,createdAt,createdById",
        }
        r = httpx.get(f"{base}/api/v1/Note", headers=headers, params=params, timeout=30)
        if r.status_code != 200:
            print(f"GET /Note failed: HTTP {r.status_code} — {r.text[:200]}", file=sys.stderr)
            return 1

        data = r.json()
        rows: list[dict[str, Any]] = data.get("list", []) or []
        if not rows:
            break

        for row in rows:
            parent_type = row.get("parentType")
            parent_id = row.get("parentId")
            post = row.get("post") or ""
            note_id = row.get("id")

            if parent_type != "Case" or not parent_id:
                skipped += 1
                continue

            parsed = parse_note_post(post)
            if not parsed:
                skipped += 1
                continue

            category, content = parsed
            name = content[:100]
            print(f"  migrate Note {note_id} [{parent_id}] → CaseMemory ({category}): {name[:60]}")

            if args.dry_run:
                migrated += 1
                continue

            create_body = {
                "name": name,
                "caseId": parent_id,
                "category": category,
                "content": content,
                "source": "assistant",
                "importance": "normal",
            }
            cr = httpx.post(f"{base}/api/v1/CaseMemory", headers=headers, json=create_body, timeout=30)
            if cr.status_code not in (200, 201):
                print(f"    CREATE CaseMemory failed: HTTP {cr.status_code} — {cr.text[:200]}", file=sys.stderr)
                failed += 1
                continue

            dr = httpx.delete(f"{base}/api/v1/Note/{note_id}", headers=headers, timeout=30)
            if dr.status_code not in (200, 204):
                print(f"    DELETE Note {note_id} failed: HTTP {dr.status_code} — {dr.text[:200]}", file=sys.stderr)
            migrated += 1

        offset += page_size
        # Don't loop forever on a misbehaving total.
        total = data.get("total", 0)
        if offset >= total:
            break
        time.sleep(0.2)  # gentle rate-limit

    print()
    print(f"=== summary ===")
    print(f"  migrated: {migrated}")
    print(f"  skipped:  {skipped}")
    print(f"  failed:   {failed}")
    print(f"  mode:     {'dry-run' if args.dry_run else 'live'}")
    return 0


if __name__ == "__main__":
    sys.exit(main())