espocrm-extensions/extension-platform
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add health check endpoint + twice-daily monitoring workflow

Browse Source 
New /health-check endpoint verifies:
- Gitea release integrity (manifest version matches tag)
- Duplicate ZIP assets in releases
- EspoCRM API reachability per customer
- Extension load status on each instance
- Stale deployments (logged ok but version not updated)

New n8n workflow runs at 07:00 & 19:00, combines health check
with drift sync, and posts combined report to Mattermost.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chaim Marcus
2026-03-30 11:38:36 +00:00
parent f546b7d660
commit 0c77056422
2 changed files with 258 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
n8n-workflows/extension-health-check.json
+128
View File
@@ -0,0 +1,128 @@
{
"name": "Extension Platform: Health & Drift Check (2x Daily)",
"nodes": [
{
"parameters": {
"rule": {
"interval": [
{ "triggerAtHour": 7 },
{ "triggerAtHour": 19 }
]
}
},
"id": "schedule-trigger",
"name": "07:00 & 19:00 Daily",
"type": "n8n-nodes-base.scheduleTrigger",
"typeVersion": 1.2,
"position": [0, 0]
},
{
"parameters": {
"method": "POST",
"url": "https://platform.dev.marcus-law.co.il/sync",
"sendBody": true,
"bodyParameters": { "parameters": [] },
"options": { "timeout": 120000 }
},
"id": "call-sync",
"name": "Sync All Customers",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [240, -100]
},
{
"parameters": {
"method": "POST",
"url": "https://platform.dev.marcus-law.co.il/health-check",
"options": { "timeout": 120000 }
},
"id": "call-health",
"name": "Health Check",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [240, 100]
},
{
"parameters": {
"mode": "wait",
"options": {}
},
"id": "merge-results",
"name": "Wait for Both",
"type": "n8n-nodes-base.merge",
"typeVersion": 3,
"position": [480, 0]
},
{
"parameters": {
"jsCode": "// Combine sync + health check results into a single Mattermost report\nconst items = $input.all();\nconst syncResult = items[0]?.json || {};\nconst healthResult = items[1]?.json || {};\n\nconst lines = [];\nconst now = new Date().toLocaleString('he-IL', { timeZone: 'Asia/Jerusalem' });\n\n// === Health Check Section ===\nconst issues = healthResult.issues || [];\nconst critical = issues.filter(i => i.severity === 'critical');\nconst errors = issues.filter(i => i.severity === 'error');\nconst warnings = issues.filter(i => i.severity === 'warning');\n\nif (issues.length > 0) {\n const statusIcon = critical.length > 0 ? ':rotating_light:' : errors.length > 0 ? ':x:' : ':warning:';\n lines.push(`### ${statusIcon} Extension Health Check — ${now}`);\n lines.push(`**${issues.length}** issues found: ${critical.length} critical, ${errors.length} errors, ${warnings.length} warnings`);\n lines.push('');\n\n if (critical.length > 0) {\n lines.push('#### :rotating_light: Critical');\n for (const i of critical) {\n lines.push(`- **${i.extension || 'System'}** [${i.check}]: ${i.message}`);\n }\n lines.push('');\n }\n if (errors.length > 0) {\n lines.push('#### :x: Errors');\n for (const i of errors) {\n lines.push(`- **${i.extension || 'System'}** [${i.check}]: ${i.message}`);\n }\n lines.push('');\n }\n if (warnings.length > 0) {\n lines.push('#### :warning: Warnings');\n for (const i of warnings) {\n lines.push(`- **${i.extension || 'System'}** [${i.check}]: ${i.message}`);\n }\n lines.push('');\n }\n} else {\n lines.push(`### :white_check_mark: Extension Health Check — ${now}`);\n lines.push('All extensions passed integrity checks.');\n lines.push('');\n}\n\n// Customer API status\nconst customers = healthResult.customers || [];\nif (customers.length > 0) {\n lines.push('| Customer | API | Extensions | Version |');\n lines.push('|----------|-----|------------|---------|');\n for (const c of customers) {\n const apiStatus = c.api_reachable ? ':white_check_mark:' : ':x:';\n const extCount = c.extensions_loaded ? `${c.extension_count} loaded` : ':x: failed';\n lines.push(`| ${c.customer} | ${apiStatus} | ${extCount} | ${c.espo_version || '-'} |`);\n }\n lines.push('');\n}\n\n// === Drift Section ===\nconst totalDrifts = syncResult.total_drifts || 0;\nconst syncResults = syncResult.results || [];\n\nif (totalDrifts > 0) {\n lines.push(`### :mag: Drift Report — ${totalDrifts} drifts`);\n for (const r of syncResults) {\n if (r.status === 'error') {\n lines.push(`#### :x: ${r.customer}: ${r.message}`);\n continue;\n }\n if (!r.drifts || r.drifts.length === 0) continue;\n lines.push(`#### ${r.customer} (${r.drift_count} drifts)`);\n lines.push('| Extension | Type | Platform | Server |');\n lines.push('|-----------|------|----------|--------|');\n for (const d of r.drifts) {\n const type = {\n 'version_mismatch': ':arrows_counterclockwise: Mismatch',\n 'missing_on_server': ':red_circle: Missing',\n 'extra_on_server': ':large_blue_circle: Extra'\n }[d.type] || d.type;\n lines.push(`| ${d.extension} | ${type} | ${d.platform || '-'} | ${d.server || '-'} |`);\n }\n if (r.updated > 0) lines.push(`> Updated ${r.updated} records`);\n lines.push('');\n }\n} else {\n lines.push(`---`);\n lines.push(`:white_check_mark: **Drift check**: ${syncResult.customers_checked || 0} customers, no drifts.`);\n}\n\nreturn [{ json: { message: lines.join('\\n'), has_issues: issues.length > 0 || totalDrifts > 0 } }];"
},
"id": "format-report",
"name": "Format Combined Report",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [700, 0]
},
{
"parameters": {
"method": "POST",
"url": "https://mattermost.dev.marcus-law.co.il/api/v4/posts",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"specifyBody": "json",
"jsonBody": "={{ JSON.stringify({ channel_id: 'bdikot-veichut' in '' ? 'xq9ogsbwwpdbjnny4ggaoin87e' : 'xq9ogsbwwpdbjnny4ggaoin87e', message: $json.message }) }}",
"options": {}
},
"id": "mattermost-post",
"name": "Post to Mattermost",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [920, 0],
"credentials": {
"httpHeaderAuth": {
"id": "mattermost-token",
"name": "Mattermost Token"
}
}
}
],
"connections": {
"07:00 & 19:00 Daily": {
"main": [
[
{ "node": "Sync All Customers", "type": "main", "index": 0 },
{ "node": "Health Check", "type": "main", "index": 0 }
]
]
},
"Sync All Customers": {
"main": [
[{ "node": "Wait for Both", "type": "main", "index": 0 }]
]
},
"Health Check": {
"main": [
[{ "node": "Wait for Both", "type": "main", "index": 1 }]
]
},
"Wait for Both": {
"main": [
[{ "node": "Format Combined Report", "type": "main", "index": 0 }]
]
},
"Format Combined Report": {
"main": [
[{ "node": "Post to Mattermost", "type": "main", "index": 0 }]
]
}
},
"settings": {
"executionOrder": "v1"
},
"tags": [
{ "name": "extension-platform" },
{ "name": "health-check" },
{ "name": "daily" }
]
}
server.py
+130
View File
@@ -439,6 +439,129 @@ def sync_customer_extensions(customer):
}
def health_check_all():
"""Run comprehensive health checks on all extensions and customers."""
issues = []
# 1. Check Gitea release integrity - manifest version matches tag
extensions = query("SELECT name, latest_version FROM extensions WHERE is_active")
for ext in extensions:
module = ext['name']
version = ext['latest_version']
if not version:
continue
tag = f'v{version}'
release = gitea_api('GET', f'/repos/{GITEA_ORG}/{module}/releases/tags/{tag}')
if release.get('error'):
issues.append({'check': 'gitea_release', 'extension': module, 'severity': 'error',
'message': f'Release {tag} not found on Gitea'})
continue
zip_assets = [a for a in release.get('assets', []) if a['name'].endswith('.zip')]
if not zip_assets:
issues.append({'check': 'gitea_release', 'extension': module, 'severity': 'error',
'message': f'No ZIP asset in release {tag}'})
continue
# Check for duplicate ZIP assets
if len(zip_assets) > 1:
issues.append({'check': 'duplicate_assets', 'extension': module, 'severity': 'warning',
'message': f'{len(zip_assets)} ZIP assets in release {tag} — should be 1'})
# Download last ZIP and verify manifest version matches tag
last_zip_url = zip_assets[-1]['browser_download_url']
try:
req = urllib.request.Request(last_zip_url, headers={'Authorization': f'token {GITEA_TOKEN}'})
resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=30)
zip_data = resp.read()
with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
manifest_files = [n for n in zf.namelist() if n.endswith('manifest.json')]
if manifest_files:
manifest = json.loads(zf.read(manifest_files[0]))
manifest_ver = manifest.get('version', '')
if manifest_ver != version:
issues.append({'check': 'version_mismatch', 'extension': module, 'severity': 'critical',
'message': f'ZIP manifest says {manifest_ver} but release tag is {tag} — wrong ZIP!'})
else:
issues.append({'check': 'manifest_missing', 'extension': module, 'severity': 'error',
'message': f'No manifest.json found inside ZIP for {tag}'})
except Exception as e:
issues.append({'check': 'zip_verify', 'extension': module, 'severity': 'error',
'message': f'Failed to verify ZIP: {str(e)[:150]}'})
# 2. Check EspoCRM API health for each customer
customers = query("""SELECT id, name, espocrm_url, espocrm_api_key
FROM customers WHERE is_active AND espocrm_api_key IS NOT NULL""")
customer_results = []
for c in customers:
espo_url = c['espocrm_url'].rstrip('/')
headers = {'Espo-Authorization': c['espocrm_api_key']}
cr = {'customer': c['name'], 'url': espo_url, 'api_reachable': False, 'extensions_loaded': False}
# API health
try:
req = urllib.request.Request(f'{espo_url}/api/v1/App/user', headers=headers)
resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=15)
data = json.loads(resp.read())
cr['api_reachable'] = True
cr['espo_version'] = data.get('version', 'unknown')
except Exception as e:
issues.append({'check': 'api_health', 'extension': None, 'severity': 'critical',
'message': f'{c["name"]}: EspoCRM API unreachable — {str(e)[:150]}'})
customer_results.append(cr)
continue
# Check extensions are loaded
try:
req = urllib.request.Request(f'{espo_url}/api/v1/Extension?select=name,version&maxSize=200', headers=headers)
resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=15)
data = json.loads(resp.read())
server_exts = {e['name']: e.get('version', '') for e in data.get('list', [])}
cr['extensions_loaded'] = True
cr['extension_count'] = len(server_exts)
# Check installed extensions actually match what platform expects
platform_exts = query("""SELECT e.name, ce.installed_version FROM customer_extensions ce
JOIN extensions e ON ce.extension_id = e.id
WHERE ce.customer_id = %s AND ce.installed_version IS NOT NULL""", (c['id'],))
for pe in platform_exts:
server_ver = server_exts.get(pe['name'])
if server_ver and server_ver != pe['installed_version']:
issues.append({'check': 'install_integrity', 'extension': pe['name'], 'severity': 'warning',
'message': f'{c["name"]}: platform says {pe["installed_version"]} but server has {server_ver}'})
except Exception as e:
issues.append({'check': 'ext_query', 'extension': None, 'severity': 'error',
'message': f'{c["name"]}: failed to query extensions — {str(e)[:150]}'})
customer_results.append(cr)
# 3. Check for stale deployments (status ok but installed_version not updated)
stale = query("""SELECT dl.version as deployed_version, e.name as extension, c.name as customer,
ce.installed_version, dl.started_at
FROM deployment_log dl
JOIN extensions e ON dl.extension_id = e.id
JOIN customers c ON dl.customer_id = c.id
LEFT JOIN customer_extensions ce ON ce.customer_id = dl.customer_id AND ce.extension_id = dl.extension_id
WHERE dl.status = 'ok' AND dl.started_at > NOW() - INTERVAL '24 hours'
AND (ce.installed_version IS NULL OR ce.installed_version != dl.version)
ORDER BY dl.started_at DESC""")
for s in stale:
issues.append({'check': 'stale_deploy', 'extension': s['extension'], 'severity': 'warning',
'message': f'{s["customer"]}: deployed {s["deployed_version"]} but installed shows {s["installed_version"] or "NULL"}'})
critical = len([i for i in issues if i['severity'] == 'critical'])
errors = len([i for i in issues if i['severity'] == 'error'])
warnings = len([i for i in issues if i['severity'] == 'warning'])
return {
'status': 'critical' if critical else 'error' if errors else 'warning' if warnings else 'ok',
'total_issues': len(issues),
'critical': critical, 'errors': errors, 'warnings': warnings,
'issues': issues,
'customers': customer_results
}
def sync_all_customers():
"""Run sync for all active customers."""
customers = query("""SELECT id, name, espocrm_url, espocrm_api_key
@@ -492,6 +615,8 @@ class APIHandler(BaseHTTPRequestHandler):
self._handle_deploy(body)
elif path == '/sync':
self._handle_sync(body)
elif path == '/health-check':
self._handle_health_check()
else:
self._respond(404, {'error': 'Not found'})
@@ -673,6 +798,11 @@ class APIHandler(BaseHTTPRequestHandler):
result = sync_all_customers()
self._respond(200, result)
# --- Health Check ---
def _handle_health_check(self):
result = health_check_all()
self._respond(200, result)
# --- Helpers ---
def _read_body(self):
length = int(self.headers.get('Content-Length', 0))