From 0c770564228c08555f9c59d5df524721fc390e36 Mon Sep 17 00:00:00 2001 From: Chaim Date: Mon, 30 Mar 2026 11:38:36 +0000 Subject: [PATCH] feat: add health check endpoint + twice-daily monitoring workflow New /health-check endpoint verifies: - Gitea release integrity (manifest version matches tag) - Duplicate ZIP assets in releases - EspoCRM API reachability per customer - Extension load status on each instance - Stale deployments (logged ok but version not updated) New n8n workflow runs at 07:00 & 19:00, combines health check with drift sync, and posts combined report to Mattermost. Co-Authored-By: Claude Opus 4.6 (1M context) --- n8n-workflows/extension-health-check.json | 128 +++++++++++++++++++++ server.py | 130 ++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 n8n-workflows/extension-health-check.json diff --git a/n8n-workflows/extension-health-check.json b/n8n-workflows/extension-health-check.json new file mode 100644 index 0000000..b04b77b --- /dev/null +++ b/n8n-workflows/extension-health-check.json @@ -0,0 +1,128 @@ +{ + "name": "Extension Platform: Health & Drift Check (2x Daily)", + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { "triggerAtHour": 7 }, + { "triggerAtHour": 19 } + ] + } + }, + "id": "schedule-trigger", + "name": "07:00 & 19:00 Daily", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [0, 0] + }, + { + "parameters": { + "method": "POST", + "url": "https://platform.dev.marcus-law.co.il/sync", + "sendBody": true, + "bodyParameters": { "parameters": [] }, + "options": { "timeout": 120000 } + }, + "id": "call-sync", + "name": "Sync All Customers", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [240, -100] + }, + { + "parameters": { + "method": "POST", + "url": "https://platform.dev.marcus-law.co.il/health-check", + "options": { "timeout": 120000 } + }, + "id": "call-health", + "name": "Health Check", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [240, 100] + }, + { + "parameters": { + "mode": "wait", + "options": {} + }, + "id": "merge-results", + "name": "Wait for Both", + "type": "n8n-nodes-base.merge", + "typeVersion": 3, + "position": [480, 0] + }, + { + "parameters": { + "jsCode": "// Combine sync + health check results into a single Mattermost report\nconst items = $input.all();\nconst syncResult = items[0]?.json || {};\nconst healthResult = items[1]?.json || {};\n\nconst lines = [];\nconst now = new Date().toLocaleString('he-IL', { timeZone: 'Asia/Jerusalem' });\n\n// === Health Check Section ===\nconst issues = healthResult.issues || [];\nconst critical = issues.filter(i => i.severity === 'critical');\nconst errors = issues.filter(i => i.severity === 'error');\nconst warnings = issues.filter(i => i.severity === 'warning');\n\nif (issues.length > 0) {\n const statusIcon = critical.length > 0 ? ':rotating_light:' : errors.length > 0 ? ':x:' : ':warning:';\n lines.push(`### ${statusIcon} Extension Health Check — ${now}`);\n lines.push(`**${issues.length}** issues found: ${critical.length} critical, ${errors.length} errors, ${warnings.length} warnings`);\n lines.push('');\n\n if (critical.length > 0) {\n lines.push('#### :rotating_light: Critical');\n for (const i of critical) {\n lines.push(`- **${i.extension || 'System'}** [${i.check}]: ${i.message}`);\n }\n lines.push('');\n }\n if (errors.length > 0) {\n lines.push('#### :x: Errors');\n for (const i of errors) {\n lines.push(`- **${i.extension || 'System'}** [${i.check}]: ${i.message}`);\n }\n lines.push('');\n }\n if (warnings.length > 0) {\n lines.push('#### :warning: Warnings');\n for (const i of warnings) {\n lines.push(`- **${i.extension || 'System'}** [${i.check}]: ${i.message}`);\n }\n lines.push('');\n }\n} else {\n lines.push(`### :white_check_mark: Extension Health Check — ${now}`);\n lines.push('All extensions passed integrity checks.');\n lines.push('');\n}\n\n// Customer API status\nconst customers = healthResult.customers || [];\nif (customers.length > 0) {\n lines.push('| Customer | API | Extensions | Version |');\n lines.push('|----------|-----|------------|---------|');\n for (const c of customers) {\n const apiStatus = c.api_reachable ? ':white_check_mark:' : ':x:';\n const extCount = c.extensions_loaded ? `${c.extension_count} loaded` : ':x: failed';\n lines.push(`| ${c.customer} | ${apiStatus} | ${extCount} | ${c.espo_version || '-'} |`);\n }\n lines.push('');\n}\n\n// === Drift Section ===\nconst totalDrifts = syncResult.total_drifts || 0;\nconst syncResults = syncResult.results || [];\n\nif (totalDrifts > 0) {\n lines.push(`### :mag: Drift Report — ${totalDrifts} drifts`);\n for (const r of syncResults) {\n if (r.status === 'error') {\n lines.push(`#### :x: ${r.customer}: ${r.message}`);\n continue;\n }\n if (!r.drifts || r.drifts.length === 0) continue;\n lines.push(`#### ${r.customer} (${r.drift_count} drifts)`);\n lines.push('| Extension | Type | Platform | Server |');\n lines.push('|-----------|------|----------|--------|');\n for (const d of r.drifts) {\n const type = {\n 'version_mismatch': ':arrows_counterclockwise: Mismatch',\n 'missing_on_server': ':red_circle: Missing',\n 'extra_on_server': ':large_blue_circle: Extra'\n }[d.type] || d.type;\n lines.push(`| ${d.extension} | ${type} | ${d.platform || '-'} | ${d.server || '-'} |`);\n }\n if (r.updated > 0) lines.push(`> Updated ${r.updated} records`);\n lines.push('');\n }\n} else {\n lines.push(`---`);\n lines.push(`:white_check_mark: **Drift check**: ${syncResult.customers_checked || 0} customers, no drifts.`);\n}\n\nreturn [{ json: { message: lines.join('\\n'), has_issues: issues.length > 0 || totalDrifts > 0 } }];" + }, + "id": "format-report", + "name": "Format Combined Report", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [700, 0] + }, + { + "parameters": { + "method": "POST", + "url": "https://mattermost.dev.marcus-law.co.il/api/v4/posts", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ channel_id: 'bdikot-veichut' in '' ? 'xq9ogsbwwpdbjnny4ggaoin87e' : 'xq9ogsbwwpdbjnny4ggaoin87e', message: $json.message }) }}", + "options": {} + }, + "id": "mattermost-post", + "name": "Post to Mattermost", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [920, 0], + "credentials": { + "httpHeaderAuth": { + "id": "mattermost-token", + "name": "Mattermost Token" + } + } + } + ], + "connections": { + "07:00 & 19:00 Daily": { + "main": [ + [ + { "node": "Sync All Customers", "type": "main", "index": 0 }, + { "node": "Health Check", "type": "main", "index": 0 } + ] + ] + }, + "Sync All Customers": { + "main": [ + [{ "node": "Wait for Both", "type": "main", "index": 0 }] + ] + }, + "Health Check": { + "main": [ + [{ "node": "Wait for Both", "type": "main", "index": 1 }] + ] + }, + "Wait for Both": { + "main": [ + [{ "node": "Format Combined Report", "type": "main", "index": 0 }] + ] + }, + "Format Combined Report": { + "main": [ + [{ "node": "Post to Mattermost", "type": "main", "index": 0 }] + ] + } + }, + "settings": { + "executionOrder": "v1" + }, + "tags": [ + { "name": "extension-platform" }, + { "name": "health-check" }, + { "name": "daily" } + ] +} diff --git a/server.py b/server.py index 54adddc..698e044 100644 --- a/server.py +++ b/server.py @@ -439,6 +439,129 @@ def sync_customer_extensions(customer): } +def health_check_all(): + """Run comprehensive health checks on all extensions and customers.""" + issues = [] + + # 1. Check Gitea release integrity - manifest version matches tag + extensions = query("SELECT name, latest_version FROM extensions WHERE is_active") + for ext in extensions: + module = ext['name'] + version = ext['latest_version'] + if not version: + continue + tag = f'v{version}' + release = gitea_api('GET', f'/repos/{GITEA_ORG}/{module}/releases/tags/{tag}') + if release.get('error'): + issues.append({'check': 'gitea_release', 'extension': module, 'severity': 'error', + 'message': f'Release {tag} not found on Gitea'}) + continue + + zip_assets = [a for a in release.get('assets', []) if a['name'].endswith('.zip')] + if not zip_assets: + issues.append({'check': 'gitea_release', 'extension': module, 'severity': 'error', + 'message': f'No ZIP asset in release {tag}'}) + continue + + # Check for duplicate ZIP assets + if len(zip_assets) > 1: + issues.append({'check': 'duplicate_assets', 'extension': module, 'severity': 'warning', + 'message': f'{len(zip_assets)} ZIP assets in release {tag} — should be 1'}) + + # Download last ZIP and verify manifest version matches tag + last_zip_url = zip_assets[-1]['browser_download_url'] + try: + req = urllib.request.Request(last_zip_url, headers={'Authorization': f'token {GITEA_TOKEN}'}) + resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=30) + zip_data = resp.read() + with zipfile.ZipFile(io.BytesIO(zip_data)) as zf: + manifest_files = [n for n in zf.namelist() if n.endswith('manifest.json')] + if manifest_files: + manifest = json.loads(zf.read(manifest_files[0])) + manifest_ver = manifest.get('version', '') + if manifest_ver != version: + issues.append({'check': 'version_mismatch', 'extension': module, 'severity': 'critical', + 'message': f'ZIP manifest says {manifest_ver} but release tag is {tag} — wrong ZIP!'}) + else: + issues.append({'check': 'manifest_missing', 'extension': module, 'severity': 'error', + 'message': f'No manifest.json found inside ZIP for {tag}'}) + except Exception as e: + issues.append({'check': 'zip_verify', 'extension': module, 'severity': 'error', + 'message': f'Failed to verify ZIP: {str(e)[:150]}'}) + + # 2. Check EspoCRM API health for each customer + customers = query("""SELECT id, name, espocrm_url, espocrm_api_key + FROM customers WHERE is_active AND espocrm_api_key IS NOT NULL""") + customer_results = [] + for c in customers: + espo_url = c['espocrm_url'].rstrip('/') + headers = {'Espo-Authorization': c['espocrm_api_key']} + cr = {'customer': c['name'], 'url': espo_url, 'api_reachable': False, 'extensions_loaded': False} + + # API health + try: + req = urllib.request.Request(f'{espo_url}/api/v1/App/user', headers=headers) + resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=15) + data = json.loads(resp.read()) + cr['api_reachable'] = True + cr['espo_version'] = data.get('version', 'unknown') + except Exception as e: + issues.append({'check': 'api_health', 'extension': None, 'severity': 'critical', + 'message': f'{c["name"]}: EspoCRM API unreachable — {str(e)[:150]}'}) + customer_results.append(cr) + continue + + # Check extensions are loaded + try: + req = urllib.request.Request(f'{espo_url}/api/v1/Extension?select=name,version&maxSize=200', headers=headers) + resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=15) + data = json.loads(resp.read()) + server_exts = {e['name']: e.get('version', '') for e in data.get('list', [])} + cr['extensions_loaded'] = True + cr['extension_count'] = len(server_exts) + + # Check installed extensions actually match what platform expects + platform_exts = query("""SELECT e.name, ce.installed_version FROM customer_extensions ce + JOIN extensions e ON ce.extension_id = e.id + WHERE ce.customer_id = %s AND ce.installed_version IS NOT NULL""", (c['id'],)) + for pe in platform_exts: + server_ver = server_exts.get(pe['name']) + if server_ver and server_ver != pe['installed_version']: + issues.append({'check': 'install_integrity', 'extension': pe['name'], 'severity': 'warning', + 'message': f'{c["name"]}: platform says {pe["installed_version"]} but server has {server_ver}'}) + except Exception as e: + issues.append({'check': 'ext_query', 'extension': None, 'severity': 'error', + 'message': f'{c["name"]}: failed to query extensions — {str(e)[:150]}'}) + + customer_results.append(cr) + + # 3. Check for stale deployments (status ok but installed_version not updated) + stale = query("""SELECT dl.version as deployed_version, e.name as extension, c.name as customer, + ce.installed_version, dl.started_at + FROM deployment_log dl + JOIN extensions e ON dl.extension_id = e.id + JOIN customers c ON dl.customer_id = c.id + LEFT JOIN customer_extensions ce ON ce.customer_id = dl.customer_id AND ce.extension_id = dl.extension_id + WHERE dl.status = 'ok' AND dl.started_at > NOW() - INTERVAL '24 hours' + AND (ce.installed_version IS NULL OR ce.installed_version != dl.version) + ORDER BY dl.started_at DESC""") + for s in stale: + issues.append({'check': 'stale_deploy', 'extension': s['extension'], 'severity': 'warning', + 'message': f'{s["customer"]}: deployed {s["deployed_version"]} but installed shows {s["installed_version"] or "NULL"}'}) + + critical = len([i for i in issues if i['severity'] == 'critical']) + errors = len([i for i in issues if i['severity'] == 'error']) + warnings = len([i for i in issues if i['severity'] == 'warning']) + + return { + 'status': 'critical' if critical else 'error' if errors else 'warning' if warnings else 'ok', + 'total_issues': len(issues), + 'critical': critical, 'errors': errors, 'warnings': warnings, + 'issues': issues, + 'customers': customer_results + } + + def sync_all_customers(): """Run sync for all active customers.""" customers = query("""SELECT id, name, espocrm_url, espocrm_api_key @@ -492,6 +615,8 @@ class APIHandler(BaseHTTPRequestHandler): self._handle_deploy(body) elif path == '/sync': self._handle_sync(body) + elif path == '/health-check': + self._handle_health_check() else: self._respond(404, {'error': 'Not found'}) @@ -673,6 +798,11 @@ class APIHandler(BaseHTTPRequestHandler): result = sync_all_customers() self._respond(200, result) + # --- Health Check --- + def _handle_health_check(self): + result = health_check_all() + self._respond(200, result) + # --- Helpers --- def _read_body(self): length = int(self.headers.get('Content-Length', 0))