feat: add health check endpoint + twice-daily monitoring workflow
New /health-check endpoint verifies: - Gitea release integrity (manifest version matches tag) - Duplicate ZIP assets in releases - EspoCRM API reachability per customer - Extension load status on each instance - Stale deployments (logged ok but version not updated) New n8n workflow runs at 07:00 & 19:00, combines health check with drift sync, and posts combined report to Mattermost. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -439,6 +439,129 @@ def sync_customer_extensions(customer):
|
||||
}
|
||||
|
||||
|
||||
def health_check_all():
|
||||
"""Run comprehensive health checks on all extensions and customers."""
|
||||
issues = []
|
||||
|
||||
# 1. Check Gitea release integrity - manifest version matches tag
|
||||
extensions = query("SELECT name, latest_version FROM extensions WHERE is_active")
|
||||
for ext in extensions:
|
||||
module = ext['name']
|
||||
version = ext['latest_version']
|
||||
if not version:
|
||||
continue
|
||||
tag = f'v{version}'
|
||||
release = gitea_api('GET', f'/repos/{GITEA_ORG}/{module}/releases/tags/{tag}')
|
||||
if release.get('error'):
|
||||
issues.append({'check': 'gitea_release', 'extension': module, 'severity': 'error',
|
||||
'message': f'Release {tag} not found on Gitea'})
|
||||
continue
|
||||
|
||||
zip_assets = [a for a in release.get('assets', []) if a['name'].endswith('.zip')]
|
||||
if not zip_assets:
|
||||
issues.append({'check': 'gitea_release', 'extension': module, 'severity': 'error',
|
||||
'message': f'No ZIP asset in release {tag}'})
|
||||
continue
|
||||
|
||||
# Check for duplicate ZIP assets
|
||||
if len(zip_assets) > 1:
|
||||
issues.append({'check': 'duplicate_assets', 'extension': module, 'severity': 'warning',
|
||||
'message': f'{len(zip_assets)} ZIP assets in release {tag} — should be 1'})
|
||||
|
||||
# Download last ZIP and verify manifest version matches tag
|
||||
last_zip_url = zip_assets[-1]['browser_download_url']
|
||||
try:
|
||||
req = urllib.request.Request(last_zip_url, headers={'Authorization': f'token {GITEA_TOKEN}'})
|
||||
resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=30)
|
||||
zip_data = resp.read()
|
||||
with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
|
||||
manifest_files = [n for n in zf.namelist() if n.endswith('manifest.json')]
|
||||
if manifest_files:
|
||||
manifest = json.loads(zf.read(manifest_files[0]))
|
||||
manifest_ver = manifest.get('version', '')
|
||||
if manifest_ver != version:
|
||||
issues.append({'check': 'version_mismatch', 'extension': module, 'severity': 'critical',
|
||||
'message': f'ZIP manifest says {manifest_ver} but release tag is {tag} — wrong ZIP!'})
|
||||
else:
|
||||
issues.append({'check': 'manifest_missing', 'extension': module, 'severity': 'error',
|
||||
'message': f'No manifest.json found inside ZIP for {tag}'})
|
||||
except Exception as e:
|
||||
issues.append({'check': 'zip_verify', 'extension': module, 'severity': 'error',
|
||||
'message': f'Failed to verify ZIP: {str(e)[:150]}'})
|
||||
|
||||
# 2. Check EspoCRM API health for each customer
|
||||
customers = query("""SELECT id, name, espocrm_url, espocrm_api_key
|
||||
FROM customers WHERE is_active AND espocrm_api_key IS NOT NULL""")
|
||||
customer_results = []
|
||||
for c in customers:
|
||||
espo_url = c['espocrm_url'].rstrip('/')
|
||||
headers = {'Espo-Authorization': c['espocrm_api_key']}
|
||||
cr = {'customer': c['name'], 'url': espo_url, 'api_reachable': False, 'extensions_loaded': False}
|
||||
|
||||
# API health
|
||||
try:
|
||||
req = urllib.request.Request(f'{espo_url}/api/v1/App/user', headers=headers)
|
||||
resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=15)
|
||||
data = json.loads(resp.read())
|
||||
cr['api_reachable'] = True
|
||||
cr['espo_version'] = data.get('version', 'unknown')
|
||||
except Exception as e:
|
||||
issues.append({'check': 'api_health', 'extension': None, 'severity': 'critical',
|
||||
'message': f'{c["name"]}: EspoCRM API unreachable — {str(e)[:150]}'})
|
||||
customer_results.append(cr)
|
||||
continue
|
||||
|
||||
# Check extensions are loaded
|
||||
try:
|
||||
req = urllib.request.Request(f'{espo_url}/api/v1/Extension?select=name,version&maxSize=200', headers=headers)
|
||||
resp = urllib.request.urlopen(req, context=SSL_CTX, timeout=15)
|
||||
data = json.loads(resp.read())
|
||||
server_exts = {e['name']: e.get('version', '') for e in data.get('list', [])}
|
||||
cr['extensions_loaded'] = True
|
||||
cr['extension_count'] = len(server_exts)
|
||||
|
||||
# Check installed extensions actually match what platform expects
|
||||
platform_exts = query("""SELECT e.name, ce.installed_version FROM customer_extensions ce
|
||||
JOIN extensions e ON ce.extension_id = e.id
|
||||
WHERE ce.customer_id = %s AND ce.installed_version IS NOT NULL""", (c['id'],))
|
||||
for pe in platform_exts:
|
||||
server_ver = server_exts.get(pe['name'])
|
||||
if server_ver and server_ver != pe['installed_version']:
|
||||
issues.append({'check': 'install_integrity', 'extension': pe['name'], 'severity': 'warning',
|
||||
'message': f'{c["name"]}: platform says {pe["installed_version"]} but server has {server_ver}'})
|
||||
except Exception as e:
|
||||
issues.append({'check': 'ext_query', 'extension': None, 'severity': 'error',
|
||||
'message': f'{c["name"]}: failed to query extensions — {str(e)[:150]}'})
|
||||
|
||||
customer_results.append(cr)
|
||||
|
||||
# 3. Check for stale deployments (status ok but installed_version not updated)
|
||||
stale = query("""SELECT dl.version as deployed_version, e.name as extension, c.name as customer,
|
||||
ce.installed_version, dl.started_at
|
||||
FROM deployment_log dl
|
||||
JOIN extensions e ON dl.extension_id = e.id
|
||||
JOIN customers c ON dl.customer_id = c.id
|
||||
LEFT JOIN customer_extensions ce ON ce.customer_id = dl.customer_id AND ce.extension_id = dl.extension_id
|
||||
WHERE dl.status = 'ok' AND dl.started_at > NOW() - INTERVAL '24 hours'
|
||||
AND (ce.installed_version IS NULL OR ce.installed_version != dl.version)
|
||||
ORDER BY dl.started_at DESC""")
|
||||
for s in stale:
|
||||
issues.append({'check': 'stale_deploy', 'extension': s['extension'], 'severity': 'warning',
|
||||
'message': f'{s["customer"]}: deployed {s["deployed_version"]} but installed shows {s["installed_version"] or "NULL"}'})
|
||||
|
||||
critical = len([i for i in issues if i['severity'] == 'critical'])
|
||||
errors = len([i for i in issues if i['severity'] == 'error'])
|
||||
warnings = len([i for i in issues if i['severity'] == 'warning'])
|
||||
|
||||
return {
|
||||
'status': 'critical' if critical else 'error' if errors else 'warning' if warnings else 'ok',
|
||||
'total_issues': len(issues),
|
||||
'critical': critical, 'errors': errors, 'warnings': warnings,
|
||||
'issues': issues,
|
||||
'customers': customer_results
|
||||
}
|
||||
|
||||
|
||||
def sync_all_customers():
|
||||
"""Run sync for all active customers."""
|
||||
customers = query("""SELECT id, name, espocrm_url, espocrm_api_key
|
||||
@@ -492,6 +615,8 @@ class APIHandler(BaseHTTPRequestHandler):
|
||||
self._handle_deploy(body)
|
||||
elif path == '/sync':
|
||||
self._handle_sync(body)
|
||||
elif path == '/health-check':
|
||||
self._handle_health_check()
|
||||
else:
|
||||
self._respond(404, {'error': 'Not found'})
|
||||
|
||||
@@ -673,6 +798,11 @@ class APIHandler(BaseHTTPRequestHandler):
|
||||
result = sync_all_customers()
|
||||
self._respond(200, result)
|
||||
|
||||
# --- Health Check ---
|
||||
def _handle_health_check(self):
|
||||
result = health_check_all()
|
||||
self._respond(200, result)
|
||||
|
||||
# --- Helpers ---
|
||||
def _read_body(self):
|
||||
length = int(self.headers.get('Content-Length', 0))
|
||||
|
||||
Reference in New Issue
Block a user