fix: toDb() crash on import + add snapshot before import

- Fix: replace QueryComposer::toDb() (protected) with camelCaseToUnderscore for table name resolution — was causing 500 on real import - Feat: optional database snapshot (mysqldump) before import with UI checkbox (checked by default). Snapshot saved to data/backups/snapshot-{timestamp}/ - Feat: skip entity types whose module is not installed (import resilience) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-29 20:14:08 +00:00
parent a57e23dec3
commit 078b2e6051
7 changed files with 197 additions and 30 deletions
@@ -158,6 +158,14 @@
                                <span class="text-muted"> — {{translate 'importNoFilesHint' category='messages' scope='DataMigration'}}</span>
                            </label>
                        </div>
+
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="createSnapshot" checked>
+                                {{translate 'Create snapshot before import' scope='DataMigration'}}
+                                <span class="text-muted"> — {{translate 'snapshotHint' category='messages' scope='DataMigration'}}</span>
+                            </label>
+                        </div>
                    </div>

                    <div class="btn-group">
@@ -302,6 +302,7 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
            const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
            const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
            const noFiles = this.$el.find('input[name="importNoFiles"]').is(':checked');
+            const createSnapshot = !dryRun && this.$el.find('input[name="createSnapshot"]').is(':checked');

            const confirmMsg = dryRun
                ? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
@@ -313,37 +314,72 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
                    .addClass('disabled').attr('disabled', true);
                this.$el.find('.import-progress').show();

-                Espo.Ajax.postRequest('DataMigration/action/import', {
-                    backupName: this.selectedBackup,
-                    preserveIds: preserveIds,
-                    noFiles: noFiles,
-                    skipDuplicates: skipDuplicates,
-                    dryRun: dryRun,
-                }, {timeout: 600000})
-                .then(response => {
-                    this.isImporting = false;
-                    this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
-                        .removeClass('disabled').attr('disabled', false);
-                    this.$el.find('.import-progress').hide();
+                const runImport = () => {
+                    Espo.Ajax.postRequest('DataMigration/action/import', {
+                        backupName: this.selectedBackup,
+                        preserveIds: preserveIds,
+                        noFiles: noFiles,
+                        skipDuplicates: skipDuplicates,
+                        dryRun: dryRun,
+                    }, {timeout: 600000})
+                    .then(response => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();

-                    this.importLog = response.log || [];
-                    this.renderLog('.import-log-container', this.importLog, !dryRun);
+                        this.importLog = response.log || [];
+                        this.renderLog('.import-log-container', this.importLog, !dryRun);

-                    const label = dryRun ? 'Dry run completed' : 'Import completed';
+                        const label = dryRun ? 'Dry run completed' : 'Import completed';

-                    Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
+                        Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));

-                    if (!dryRun) {
-                        this.loadStatus();
-                    }
-                })
-                .catch(xhr => {
-                    this.isImporting = false;
-                    this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
-                        .removeClass('disabled').attr('disabled', false);
-                    this.$el.find('.import-progress').hide();
-                    Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
-                });
+                        if (!dryRun) {
+                            this.loadStatus();
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
+                    });
+                };
+
+                if (createSnapshot) {
+                    Espo.Ui.notify(this.translate('Creating snapshot...', 'labels', 'DataMigration'));
+
+                    Espo.Ajax.postRequest('DataMigration/action/snapshot', {}, {timeout: 300000})
+                    .then(response => {
+                        if (response.success) {
+                            Espo.Ui.success(
+                                this.translate('Snapshot created', 'labels', 'DataMigration') +
+                                ': ' + response.snapshotName + ' (' + response.dumpSizeFormatted + ')'
+                            );
+                            runImport();
+                        } else {
+                            this.isImporting = false;
+                            this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                                .removeClass('disabled').attr('disabled', false);
+                            this.$el.find('.import-progress').hide();
+                            Espo.Ui.error(
+                                this.translate('Snapshot failed', 'labels', 'DataMigration') +
+                                ': ' + (response.error || 'Unknown error')
+                            );
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Snapshot failed', 'labels', 'DataMigration'));
+                    });
+                } else {
+                    runImport();
+                }
            });
        },

@@ -0,0 +1,110 @@
+<?php
+
+namespace Espo\Modules\DataMigration\Api;
+
+use Espo\Core\Api\Action;
+use Espo\Core\Api\Request;
+use Espo\Core\Api\Response;
+use Espo\Core\Api\ResponseComposer;
+use Espo\Core\Exceptions\Forbidden;
+use Espo\Core\Utils\Config;
+use Espo\Entities\User;
+
+class Snapshot implements Action
+{
+    public function __construct(
+        private User $user,
+        private Config $config
+    ) {}
+
+    public function process(Request $request): Response
+    {
+        if (!$this->user->isAdmin()) {
+            throw new Forbidden();
+        }
+
+        $timestamp = date('Ymd-His');
+        $snapshotName = 'snapshot-' . $timestamp;
+        $snapshotDir = 'data/backups/' . $snapshotName;
+
+        mkdir($snapshotDir, 0755, true);
+
+        $dbParams = $this->config->get('database');
+        $host = $dbParams['host'] ?? 'localhost';
+        $port = $dbParams['port'] ?? '3306';
+        $dbName = $dbParams['dbname'] ?? '';
+        $user = $dbParams['user'] ?? '';
+        $password = $dbParams['password'] ?? '';
+
+        $dumpFile = $snapshotDir . '/database.sql';
+
+        $cmd = sprintf(
+            'mysqldump --host=%s --port=%s --user=%s --password=%s --single-transaction --quick %s > %s 2>&1',
+            escapeshellarg($host),
+            escapeshellarg($port),
+            escapeshellarg($user),
+            escapeshellarg($password),
+            escapeshellarg($dbName),
+            escapeshellarg($dumpFile)
+        );
+
+        $output = [];
+        $returnCode = 0;
+        exec($cmd, $output, $returnCode);
+
+        if ($returnCode !== 0) {
+            // Cleanup on failure
+            if (file_exists($dumpFile)) {
+                unlink($dumpFile);
+            }
+
+            rmdir($snapshotDir);
+
+            return ResponseComposer::json([
+                'success' => false,
+                'error' => 'Database dump failed: ' . implode("\n", $output),
+            ]);
+        }
+
+        $dumpSize = filesize($dumpFile);
+
+        // Write a manifest
+        $manifest = [
+            'type' => 'snapshot',
+            'createdAt' => date('c'),
+            'sourceHost' => $this->config->get('siteUrl', 'unknown'),
+            'espoVersion' => $this->config->get('version', 'unknown'),
+            'database' => $dbName,
+            'dumpSizeBytes' => $dumpSize,
+        ];
+
+        file_put_contents(
+            $snapshotDir . '/manifest.json',
+            json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
+        );
+
+        return ResponseComposer::json([
+            'success' => true,
+            'snapshotName' => $snapshotName,
+            'dumpSize' => $dumpSize,
+            'dumpSizeFormatted' => $this->formatBytes($dumpSize),
+        ]);
+    }
+
+    private function formatBytes(int $bytes): string
+    {
+        if ($bytes >= 1073741824) {
+            return round($bytes / 1073741824, 2) . ' GB';
+        }
+
+        if ($bytes >= 1048576) {
+            return round($bytes / 1048576, 2) . ' MB';
+        }
+
+        if ($bytes >= 1024) {
+            return round($bytes / 1024, 2) . ' KB';
+        }
+
+        return $bytes . ' B';
+    }
+}
@@ -25,6 +25,10 @@
        "Exporting...": "Exporting...",
        "Importing...": "Importing...",
        "Log": "Log",
+        "Create snapshot before import": "Create snapshot before import",
+        "Creating snapshot...": "Creating database snapshot...",
+        "Snapshot created": "Snapshot created",
+        "Snapshot failed": "Snapshot failed",
        "records": "records",
        "files": "files",
        "optional": "optional",
@@ -43,6 +47,7 @@
        "noFilesHint": "faster export, without attachment files",
        "preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
        "importNoFilesHint": "import only database records, skip attachment files",
+        "snapshotHint": "backup the entire database before importing, so you can restore if needed",
        "skipDuplicatesHint": "skip records that already exist in the database"
    }
 }
@@ -25,6 +25,10 @@
        "Exporting...": "מייצא...",
        "Importing...": "מייבא...",
        "Log": "לוג",
+        "Create snapshot before import": "צור צילום מצב לפני ייבוא",
+        "Creating snapshot...": "יוצר צילום מצב של מסד הנתונים...",
+        "Snapshot created": "צילום מצב נוצר",
+        "Snapshot failed": "יצירת צילום מצב נכשלה",
        "records": "רשומות",
        "files": "קבצים",
        "optional": "אופציונלי",
@@ -43,6 +47,7 @@
        "noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
        "preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
        "importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
+        "snapshotHint": "גיבוי מלא של מסד הנתונים לפני הייבוא, כדי שתוכל לשחזר במקרה הצורך",
        "skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
    }
 }
@@ -23,5 +23,10 @@
        "route": "/DataMigration/action/deleteBackup",
        "method": "post",
        "actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
+    },
+    {
+        "route": "/DataMigration/action/snapshot",
+        "method": "post",
+        "actionClassName": "Espo\\Modules\\DataMigration\\Api\\Snapshot"
    }
 ]
@@ -311,9 +311,7 @@ class EntityImporter

    private function insertWithRawSql(string $entityType, array $record, string $newId): void
    {
-        $tableName = $this->entityManager
-            ->getQueryComposer()
-            ->toDb($entityType);
+        $tableName = $this->camelCaseToUnderscore($entityType);

        // Convert camelCase fields to snake_case for SQL
        $columns = [];