release: v1.2.2

When importing entities with autoincrement (Case, Charge, Invoice),
validate each column exists in the target table before inserting.
Columns from modules not installed on the target are silently skipped
and reported in the import log summary.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

files/client/custom/modules/data-migration/res/templates/admin/data-migration.tpl

@@ -150,6 +150,22 @@
                                 <span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
                             </label>
                         </div>
+
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="importNoFiles">
+                                {{translate 'Skip attachment files' scope='DataMigration'}}
+                                <span class="text-muted"> — {{translate 'importNoFilesHint' category='messages' scope='DataMigration'}}</span>
+                            </label>
+                        </div>
+
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="createSnapshot" checked>
+                                {{translate 'Create snapshot before import' scope='DataMigration'}}
+                                <span class="text-muted"> — {{translate 'snapshotHint' category='messages' scope='DataMigration'}}</span>
+                            </label>
+                        </div>
                     </div>
 
                     <div class="btn-group">

files/client/custom/modules/data-migration/src/views/admin/data-migration.js

@@ -301,6 +301,8 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
 
             const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
             const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
+            const noFiles = this.$el.find('input[name="importNoFiles"]').is(':checked');
+            const createSnapshot = !dryRun && this.$el.find('input[name="createSnapshot"]').is(':checked');
 
             const confirmMsg = dryRun
                 ? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
@@ -312,36 +314,72 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
                     .addClass('disabled').attr('disabled', true);
                 this.$el.find('.import-progress').show();
 
-                Espo.Ajax.postRequest('DataMigration/action/import', {
-                    backupName: this.selectedBackup,
-                    preserveIds: preserveIds,
-                    skipDuplicates: skipDuplicates,
-                    dryRun: dryRun,
-                }, {timeout: 600000})
-                .then(response => {
-                    this.isImporting = false;
-                    this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
-                        .removeClass('disabled').attr('disabled', false);
-                    this.$el.find('.import-progress').hide();
+                const runImport = () => {
+                    Espo.Ajax.postRequest('DataMigration/action/import', {
+                        backupName: this.selectedBackup,
+                        preserveIds: preserveIds,
+                        noFiles: noFiles,
+                        skipDuplicates: skipDuplicates,
+                        dryRun: dryRun,
+                    }, {timeout: 600000})
+                    .then(response => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
 
-                    this.importLog = response.log || [];
-                    this.renderLog('.import-log-container', this.importLog, !dryRun);
+                        this.importLog = response.log || [];
+                        this.renderLog('.import-log-container', this.importLog, !dryRun);
 
-                    const label = dryRun ? 'Dry run completed' : 'Import completed';
+                        const label = dryRun ? 'Dry run completed' : 'Import completed';
 
-                    Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
+                        Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
 
-                    if (!dryRun) {
-                        this.loadStatus();
-                    }
-                })
-                .catch(xhr => {
-                    this.isImporting = false;
-                    this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
-                        .removeClass('disabled').attr('disabled', false);
-                    this.$el.find('.import-progress').hide();
-                    Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
-                });
+                        if (!dryRun) {
+                            this.loadStatus();
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
+                    });
+                };
+
+                if (createSnapshot) {
+                    Espo.Ui.notify(this.translate('Creating snapshot...', 'labels', 'DataMigration'));
+
+                    Espo.Ajax.postRequest('DataMigration/action/snapshot', {}, {timeout: 300000})
+                    .then(response => {
+                        if (response.success) {
+                            Espo.Ui.success(
+                                this.translate('Snapshot created', 'labels', 'DataMigration') +
+                                ': ' + response.snapshotName + ' (' + response.dumpSizeFormatted + ')'
+                            );
+                            runImport();
+                        } else {
+                            this.isImporting = false;
+                            this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                                .removeClass('disabled').attr('disabled', false);
+                            this.$el.find('.import-progress').hide();
+                            Espo.Ui.error(
+                                this.translate('Snapshot failed', 'labels', 'DataMigration') +
+                                ': ' + (response.error || 'Unknown error')
+                            );
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Snapshot failed', 'labels', 'DataMigration'));
+                    });
+                } else {
+                    runImport();
+                }
             });
         },
 

files/custom/Espo/Modules/DataMigration/Api/Import.php

@@ -46,6 +46,7 @@ class Import implements Action
         $preserveIds = $body->preserveIds ?? true;
         $skipDuplicates = $body->skipDuplicates ?? true;
         $dryRun = $body->dryRun ?? false;
+        $noFiles = $body->noFiles ?? false;
 
         $io = new BufferedIO();
 
@@ -54,6 +55,7 @@ class Import implements Action
             'preserveIds' => $preserveIds,
             'skipDuplicates' => $skipDuplicates,
             'dryRun' => $dryRun,
+            'noFiles' => $noFiles,
             'verbose' => true,
         ], $io);
 

files/custom/Espo/Modules/DataMigration/Api/Snapshot.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace Espo\Modules\DataMigration\Api;
+
+use Espo\Core\Api\Action;
+use Espo\Core\Api\Request;
+use Espo\Core\Api\Response;
+use Espo\Core\Api\ResponseComposer;
+use Espo\Core\Exceptions\Forbidden;
+use Espo\Core\Utils\Config;
+use Espo\Entities\User;
+
+class Snapshot implements Action
+{
+    public function __construct(
+        private User $user,
+        private Config $config
+    ) {}
+
+    public function process(Request $request): Response
+    {
+        if (!$this->user->isAdmin()) {
+            throw new Forbidden();
+        }
+
+        $timestamp = date('Ymd-His');
+        $snapshotName = 'snapshot-' . $timestamp;
+        $snapshotDir = 'data/backups/' . $snapshotName;
+
+        mkdir($snapshotDir, 0755, true);
+
+        $dbParams = $this->config->get('database');
+        $host = $dbParams['host'] ?? 'localhost';
+        $port = $dbParams['port'] ?? '3306';
+        $dbName = $dbParams['dbname'] ?? '';
+        $user = $dbParams['user'] ?? '';
+        $password = $dbParams['password'] ?? '';
+
+        $dumpFile = $snapshotDir . '/database.sql';
+
+        $cmd = sprintf(
+            'mysqldump --host=%s --port=%s --user=%s --password=%s'
+            . ' --ssl=false --no-tablespaces --single-transaction --quick %s > %s 2>&1',
+            escapeshellarg($host),
+            escapeshellarg($port),
+            escapeshellarg($user),
+            escapeshellarg($password),
+            escapeshellarg($dbName),
+            escapeshellarg($dumpFile)
+        );
+
+        $output = [];
+        $returnCode = 0;
+        exec($cmd, $output, $returnCode);
+
+        if ($returnCode !== 0) {
+            // Cleanup on failure
+            if (file_exists($dumpFile)) {
+                unlink($dumpFile);
+            }
+
+            rmdir($snapshotDir);
+
+            return ResponseComposer::json([
+                'success' => false,
+                'error' => 'Database dump failed: ' . implode("\n", $output),
+            ]);
+        }
+
+        $dumpSize = filesize($dumpFile);
+
+        // Write a manifest
+        $manifest = [
+            'type' => 'snapshot',
+            'createdAt' => date('c'),
+            'sourceHost' => $this->config->get('siteUrl', 'unknown'),
+            'espoVersion' => $this->config->get('version', 'unknown'),
+            'database' => $dbName,
+            'dumpSizeBytes' => $dumpSize,
+        ];
+
+        file_put_contents(
+            $snapshotDir . '/manifest.json',
+            json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
+        );
+
+        return ResponseComposer::json([
+            'success' => true,
+            'snapshotName' => $snapshotName,
+            'dumpSize' => $dumpSize,
+            'dumpSizeFormatted' => $this->formatBytes($dumpSize),
+        ]);
+    }
+
+    private function formatBytes(int $bytes): string
+    {
+        if ($bytes >= 1073741824) {
+            return round($bytes / 1073741824, 2) . ' GB';
+        }
+
+        if ($bytes >= 1048576) {
+            return round($bytes / 1048576, 2) . ' MB';
+        }
+
+        if ($bytes >= 1024) {
+            return round($bytes / 1024, 2) . ' KB';
+        }
+
+        return $bytes . ' B';
+    }
+}

files/custom/Espo/Modules/DataMigration/Resources/i18n/en_US/DataMigration.json

@@ -25,6 +25,10 @@
         "Exporting...": "Exporting...",
         "Importing...": "Importing...",
         "Log": "Log",
+        "Create snapshot before import": "Create snapshot before import",
+        "Creating snapshot...": "Creating database snapshot...",
+        "Snapshot created": "Snapshot created",
+        "Snapshot failed": "Snapshot failed",
         "records": "records",
         "files": "files",
         "optional": "optional",
@@ -42,6 +46,8 @@
         "deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
         "noFilesHint": "faster export, without attachment files",
         "preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
+        "importNoFilesHint": "import only database records, skip attachment files",
+        "snapshotHint": "backup the entire database before importing, so you can restore if needed",
         "skipDuplicatesHint": "skip records that already exist in the database"
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/i18n/fa_IR/Admin.json

@@ -1,6 +1,7 @@
 {
     "labels": {
-        "Data Migration": "העברת נתונים"
+        "Data Migration": "העברת נתונים",
+        "Custom Development": "פיתוח עצמאי"
     },
     "descriptions": {
         "dataMigration": "ייצוא וייבוא נתוני תיקים בין מערכות EspoCRM"

files/custom/Espo/Modules/DataMigration/Resources/i18n/fa_IR/DataMigration.json

@@ -25,6 +25,10 @@
         "Exporting...": "מייצא...",
         "Importing...": "מייבא...",
         "Log": "לוג",
+        "Create snapshot before import": "צור צילום מצב לפני ייבוא",
+        "Creating snapshot...": "יוצר צילום מצב של מסד הנתונים...",
+        "Snapshot created": "צילום מצב נוצר",
+        "Snapshot failed": "יצירת צילום מצב נכשלה",
         "records": "רשומות",
         "files": "קבצים",
         "optional": "אופציונלי",
@@ -42,6 +46,8 @@
         "deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
         "noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
         "preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
+        "importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
+        "snapshotHint": "גיבוי מלא של מסד הנתונים לפני הייבוא, כדי שתוכל לשחזר במקרה הצורך",
         "skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/metadata/app/adminPanel.json

@@ -1,14 +1,17 @@
 {
-    "system": {
+    "customDev": {
+        "label": "Custom Development",
         "itemList": [
+            "__APPEND__",
             {
                 "url": "#Admin/dataMigration",
                 "label": "Data Migration",
                 "iconClass": "fas fa-exchange-alt",
                 "description": "dataMigration",
                 "view": "modules/data-migration/views/admin/data-migration",
-                "order": 85
+                "order": 50
             }
-        ]
+        ],
+        "order": 30
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/routes.json

@@ -23,5 +23,10 @@
         "route": "/DataMigration/action/deleteBackup",
         "method": "post",
         "actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
+    },
+    {
+        "route": "/DataMigration/action/snapshot",
+        "method": "post",
+        "actionClassName": "Espo\\Modules\\DataMigration\\Api\\Snapshot"
     }
 ]

files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php

@@ -309,11 +309,16 @@ class EntityImporter
         return $newId;
     }
 
+    /** @var array<string, string[]> Cache of existing columns per table */
+    private array $tableColumnsCache = [];
+
+    /** @var array<string, string[]> Skipped columns log per entity type */
+    private array $skippedColumnsLog = [];
+
     private function insertWithRawSql(string $entityType, array $record, string $newId): void
     {
-        $tableName = $this->entityManager
-            ->getQueryComposer()
-            ->toDb($entityType);
+        $tableName = $this->camelCaseToUnderscore($entityType);
+        $existingColumns = $this->getTableColumns($tableName);
 
         // Convert camelCase fields to snake_case for SQL
         $columns = [];
@@ -327,15 +332,22 @@ class EntityImporter
                 continue;
             }
 
-            // Skip non-storable array/object fields that come from getValueMap
-            if (is_array($value) || is_object($value)) {
-                // JSON fields should be stored as JSON strings
-                if (is_array($value) || is_object($value)) {
-                    $value = json_encode($value, JSON_UNESCAPED_UNICODE);
+            $column = $this->camelCaseToUnderscore($field);
+
+            // Skip columns that don't exist in the target table
+            if (!in_array($column, $existingColumns)) {
+                if (!isset($this->skippedColumnsLog[$entityType][$column])) {
+                    $this->skippedColumnsLog[$entityType][$column] = true;
                 }
+
+                continue;
+            }
+
+            // JSON fields should be stored as JSON strings
+            if (is_array($value) || is_object($value)) {
+                $value = json_encode($value, JSON_UNESCAPED_UNICODE);
             }
 
-            $column = $this->camelCaseToUnderscore($field);
             $columns[] = "`{$column}`";
             $values[] = $value;
             $placeholders[] = '?';
@@ -346,14 +358,42 @@ class EntityImporter
 
         $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
 
-        try {
-            $pdo = $this->entityManager->getPDO();
-            $stmt = $pdo->prepare($sql);
-            $stmt->execute($values);
-        } catch (\Exception $e) {
-            // If column mismatch, try with just known columns
-            throw $e;
+        $pdo = $this->entityManager->getPDO();
+        $stmt = $pdo->prepare($sql);
+        $stmt->execute($values);
+    }
+
+    private function getTableColumns(string $tableName): array
+    {
+        if (isset($this->tableColumnsCache[$tableName])) {
+            return $this->tableColumnsCache[$tableName];
         }
+
+        $pdo = $this->entityManager->getPDO();
+        $sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
+        $columns = [];
+
+        while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
+            $columns[] = $row['Field'];
+        }
+
+        $this->tableColumnsCache[$tableName] = $columns;
+
+        return $columns;
+    }
+
+    /**
+     * @return array<string, string[]> entityType => [skipped column names]
+     */
+    public function getSkippedColumns(): array
+    {
+        $result = [];
+
+        foreach ($this->skippedColumnsLog as $entityType => $columns) {
+            $result[$entityType] = array_keys($columns);
+        }
+
+        return $result;
     }
 
     private function resetAutoIncrement(string $entityType): void

files/custom/Espo/Modules/DataMigration/Services/ImportService.php

@@ -62,6 +62,7 @@ class ImportService
         $preserveIds = $options['preserveIds'] ?? true;
         $skipDuplicates = $options['skipDuplicates'] ?? false;
         $dryRun = $options['dryRun'] ?? false;
+        $noFiles = $options['noFiles'] ?? false;
         $verbose = $options['verbose'] ?? false;
 
         $dataDir = rtrim($inputDir, '/') . '/data';
@@ -100,6 +101,11 @@ class ImportService
                 continue;
             }
 
+            if (!$this->entityTypeExists($entityType)) {
+                $io->writeLine("Skipping {$entityType} (module not installed)");
+                continue;
+            }
+
             $io->writeLine("Importing {$entityType}...");
 
             $result = $this->entityImporter->import(
@@ -133,7 +139,10 @@ class ImportService
         }
 
         // Restore attachment files
-        if (is_dir($filesDir)) {
+        if ($noFiles) {
+            $io->writeLine('');
+            $io->writeLine('Attachment files: SKIPPED (--no-files)');
+        } else if (is_dir($filesDir)) {
             $io->writeLine('');
             $io->writeLine('Restoring attachment files...');
 
@@ -170,5 +179,27 @@ class ImportService
                 }
             }
         }
+
+        // Report skipped columns (missing in target DB)
+        $skippedColumns = $this->entityImporter->getSkippedColumns();
+
+        if (!empty($skippedColumns)) {
+            $io->writeLine('');
+            $io->writeLine('WARNING: Skipped columns not found in target database:');
+
+            foreach ($skippedColumns as $type => $columns) {
+                $io->writeLine("  {$type}: " . implode(', ', $columns));
+            }
+        }
+    }
+
+    private function entityTypeExists(string $entityType): bool
+    {
+        try {
+            $this->entityManager->getRDBRepository($entityType);
+            return true;
+        } catch (\Exception $e) {
+            return false;
+        }
     }
 }

manifest.json

@@ -1,6 +1,6 @@
 {
     "name": "DataMigration",
-    "version": "1.1.0",
+    "version": "1.2.2",
     "acceptableVersions": [
         ">=8.0.0"
     ],