feat: auto-discover extensions from Gitea, sync registry with DB

MODULES was a static dict that got out of sync with Gitea repos —
LegalAssistance was missing, 5 deleted extensions were still listed.

Now MODULES loads from DB on startup with static fallback.
New POST /sync-registry endpoint scans espocrm-extensions org on Gitea,
adds new extensions, deactivates removed ones, and reloads MODULES.
INSTALL_ORDER is now computed dynamically via topological sort.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

server.py

@@ -9,6 +9,7 @@ Pipeline endpoints (called by n8n):
   POST /deploy           — Deploy extension to a specific customer via EspoCRM API
   POST /deploy-all       — Deploy to all licensed customers
   POST /sync             — Sync installed extensions from EspoCRM servers with platform DB
+  POST /sync-registry    — Discover extensions from Gitea org and sync with DB
 
 Dashboard API:
   GET  /api/stats
@@ -40,29 +41,21 @@ GITEA_ORG = os.environ.get('GITEA_ORG', 'espocrm-extensions')
 
 DIST_DIR = os.environ.get('DIST_DIR', '/app/dashboard/dist')
 
-# Module name mapping
+# Repos in the Gitea org that are NOT extensions (tools, base images, etc.)
-MODULES = {
+GITEA_NON_EXTENSION_REPOS = {'espocrm-base', 'espo-core', 'espo-tools', 'extension-dashboard', 'extension-platform'}
-    "CrmAssistant": "crm-assistant", "DigitalSignature": "digital-signature",
-    "EmailDefaults": "email-defaults", "GoogleIntegration": "google-integration",
-    "GreenInvoiceBilling": "green-invoice-billing", "HebrewLanguage": "hebrew-language",
-    "KlearBranding": "klear-branding", "LegalCrm": "legal-crm",
-    "LicenseManager": "license-manager", "LocalDocuments": "local-documents",
-    "MpdfEngine": "mpdf-engine", "NetHamishpat": "net-hamishpat",
-    "NetworkStorageIntegration": "network-storage-integration",
-    "NextCloudIntegration": "nextcloud-integration", "OfficeAssistant": "office-assistant",
-    "PhoneIsraelFormat": "phone-israel-format", "SmartAssistant": "smart-assistant",
-    "SmsIntegration": "sms-integration", "WhatsAppIntegration": "whatsapp-integration",
-    "DataMigration": "data-migration",
-}
 
-# Extension dependency tree and install order
+def _pascal_to_kebab(name):
+    """Convert PascalCase to kebab-case: LegalAssistance -> legal-assistance"""
+    return re.sub(r'(?<!^)(?=[A-Z])', '-', name).lower()
+
+# Module name mapping — loaded from DB on startup, synced from Gitea via /sync-registry
+MODULES = {}
+
+# Extension dependency tree (domain knowledge, not auto-discoverable)
 DEPENDENCIES = {
     'HebrewLanguage': [],
     'KlearBranding': ['HebrewLanguage'],
-    'PhoneIsraelFormat': ['HebrewLanguage'],
     'EmailDefaults': ['HebrewLanguage'],
-    'MpdfEngine': ['HebrewLanguage'],
-    'LicenseManager': ['HebrewLanguage'],
     'LocalDocuments': ['HebrewLanguage'],
     'GoogleIntegration': ['HebrewLanguage'],
     'NextCloudIntegration': ['HebrewLanguage'],
@@ -70,25 +63,108 @@ DEPENDENCIES = {
     'SmsIntegration': ['HebrewLanguage'],
     'WhatsAppIntegration': ['HebrewLanguage'],
     'GreenInvoiceBilling': ['HebrewLanguage'],
-    'CrmAssistant': ['HebrewLanguage'],
-    'OfficeAssistant': ['HebrewLanguage'],
     'LegalCrm': ['HebrewLanguage'],
     'NetHamishpat': ['HebrewLanguage', 'LegalCrm'],
     'DigitalSignature': ['HebrewLanguage', 'LegalCrm'],
     'SmartAssistant': ['HebrewLanguage', 'LegalCrm'],
+    'LegalAssistance': ['HebrewLanguage', 'LegalCrm'],
     'DataMigration': [],
 }
 
-INSTALL_ORDER = [
+def _compute_install_order():
-    'HebrewLanguage',
+    """Compute install order from DEPENDENCIES via topological sort. Extensions not in DEPENDENCIES are appended at the end."""
-    'KlearBranding', 'PhoneIsraelFormat', 'EmailDefaults', 'MpdfEngine',
+    order = []
-    'LicenseManager', 'LocalDocuments', 'GoogleIntegration',
+    visited = set()
-    'NextCloudIntegration', 'NetworkStorageIntegration',
+    def visit(name):
-    'SmsIntegration', 'WhatsAppIntegration', 'GreenInvoiceBilling',
+        if name in visited:
-    'CrmAssistant', 'OfficeAssistant', 'LegalCrm',
+            return
-    'NetHamishpat', 'DigitalSignature', 'SmartAssistant',
+        visited.add(name)
-    'DataMigration',
+        for dep in DEPENDENCIES.get(name, []):
-]
+            visit(dep)
+        order.append(name)
+    for name in DEPENDENCIES:
+        visit(name)
+    # Append any MODULES not in DEPENDENCIES (no deps assumed)
+    for name in sorted(MODULES):
+        if name not in visited:
+            order.append(name)
+    return order
+
+def _load_modules_from_db():
+    """Load MODULES dict from the extensions table."""
+    global MODULES
+    try:
+        rows = query("SELECT name, composer_name FROM extensions WHERE is_active")
+        if rows:
+            MODULES = {
+                r['name']: r['composer_name'].split('/')[-1] if r.get('composer_name') else _pascal_to_kebab(r['name'])
+                for r in rows
+            }
+            print(f"[init] Loaded {len(MODULES)} modules from database")
+    except Exception as e:
+        print(f"[init] Could not load modules from DB: {e}")
+
+# Static fallback used only if DB is not available on startup
+_MODULES_FALLBACK = {
+    "DataMigration": "data-migration", "DigitalSignature": "digital-signature",
+    "EmailDefaults": "email-defaults", "GoogleIntegration": "google-integration",
+    "GreenInvoiceBilling": "green-invoice-billing", "HebrewLanguage": "hebrew-language",
+    "KlearBranding": "klear-branding", "LegalAssistance": "legal-assistance",
+    "LegalCrm": "legal-crm", "LocalDocuments": "local-documents",
+    "NetHamishpat": "net-hamishpat", "NetworkStorageIntegration": "network-storage-integration",
+    "NextCloudIntegration": "nextcloud-integration", "SmartAssistant": "smart-assistant",
+    "SmsIntegration": "sms-integration", "WhatsAppIntegration": "whatsapp-integration",
+}
+
+
+def sync_extension_registry():
+    """Discover extensions from Gitea org and sync with database.
+    Adds new extensions, reactivates returned ones, deactivates removed ones."""
+    repos = gitea_api('GET', f'/orgs/{GITEA_ORG}/repos?limit=100')
+    if isinstance(repos, dict) and repos.get('error'):
+        return {'status': 'error', 'message': f'Failed to list Gitea repos: {repos.get("message", "")}'}
+
+    # Filter to extension repos: PascalCase names not in the skip list
+    gitea_extensions = {}
+    for repo in repos:
+        name = repo['name']
+        if name in GITEA_NON_EXTENSION_REPOS or not name[0].isupper():
+            continue
+        gitea_extensions[name] = _pascal_to_kebab(name)
+
+    # Get current DB state
+    db_extensions = query("SELECT name, is_active FROM extensions")
+    db_names = {r['name'] for r in db_extensions}
+    db_active = {r['name'] for r in db_extensions if r['is_active']}
+
+    added, reactivated, deactivated = [], [], []
+
+    for name, kebab in gitea_extensions.items():
+        composer_name = f'{GITEA_ORG}/{kebab}'
+        if name not in db_names:
+            query("INSERT INTO extensions (name, composer_name, is_active) VALUES (%s, %s, true)",
+                  (name, composer_name), fetch=None)
+            added.append(name)
+        elif name not in db_active:
+            query("UPDATE extensions SET is_active = true, updated_at = NOW() WHERE name = %s",
+                  (name,), fetch=None)
+            reactivated.append(name)
+
+    for name in db_active:
+        if name not in gitea_extensions:
+            query("UPDATE extensions SET is_active = false, updated_at = NOW() WHERE name = %s",
+                  (name,), fetch=None)
+            deactivated.append(name)
+
+    # Reload MODULES from DB
+    _load_modules_from_db()
+
+    return {
+        'status': 'ok',
+        'gitea_extensions': len(gitea_extensions),
+        'added': added, 'reactivated': reactivated, 'deactivated': deactivated,
+        'active_modules': sorted(MODULES.keys())
+    }
 
 SSL_CTX = ssl.create_default_context()
 SSL_CTX.check_hostname = False
@@ -678,6 +754,8 @@ class APIHandler(BaseHTTPRequestHandler):
             self._handle_sync(body)
         elif path == '/health-check':
             self._handle_health_check()
+        elif path == '/sync-registry':
+            self._handle_sync_registry()
         else:
             self._respond(404, {'error': 'Not found'})
 
@@ -732,7 +810,7 @@ class APIHandler(BaseHTTPRequestHandler):
             SELECT e.id, e.name, e.composer_name, e.latest_version, ce.installed_version, ce.license_status, ce.auto_update, ce.installed_at
             FROM customer_extensions ce JOIN extensions e ON ce.extension_id = e.id WHERE ce.customer_id = %s ORDER BY e.name""", (cid,))
         customer['dependencies'] = DEPENDENCIES
-        customer['install_order'] = INSTALL_ORDER
+        customer['install_order'] = _compute_install_order()
         self._respond(200, customer)
 
     def _create_customer(self, body):
@@ -868,6 +946,11 @@ class APIHandler(BaseHTTPRequestHandler):
         result = health_check_all()
         self._respond(200, result)
 
+    # --- Registry Sync ---
+    def _handle_sync_registry(self):
+        result = sync_extension_registry()
+        self._respond(200, result)
+
     # --- Helpers ---
     def _read_body(self):
         length = int(self.headers.get('Content-Length', 0))
@@ -910,6 +993,11 @@ class APIHandler(BaseHTTPRequestHandler):
 
 
 if __name__ == '__main__':
+    # Load modules from DB; fall back to static list if DB unavailable
+    _load_modules_from_db()
+    if not MODULES:
+        MODULES.update(_MODULES_FALLBACK)
+        print(f"[init] Using static fallback: {len(MODULES)} modules")
     port = int(os.environ.get('PORT', '8080'))
     server = HTTPServer(('0.0.0.0', port), APIHandler)
     print(f"Extension Platform API on port {port}")