release: v1.2.4

Read column metadata (type, nullable, default) from SHOW COLUMNS and
coerce values accordingly: empty string/false → 0 for NOT NULL numeric
columns, null for nullable ones. Fixes is_internal import error.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

files/client/custom/modules/data-migration/res/templates/admin/data-migration.tpl

@@ -158,6 +158,14 @@
                                 <span class="text-muted"> — {{translate 'importNoFilesHint' category='messages' scope='DataMigration'}}</span>
                             </label>
                         </div>
+
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="createSnapshot" checked>
+                                {{translate 'Create snapshot before import' scope='DataMigration'}}
+                                <span class="text-muted"> — {{translate 'snapshotHint' category='messages' scope='DataMigration'}}</span>
+                            </label>
+                        </div>
                     </div>
 
                     <div class="btn-group">

files/client/custom/modules/data-migration/src/views/admin/data-migration.js

@@ -302,6 +302,7 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
             const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
             const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
             const noFiles = this.$el.find('input[name="importNoFiles"]').is(':checked');
+            const createSnapshot = !dryRun && this.$el.find('input[name="createSnapshot"]').is(':checked');
 
             const confirmMsg = dryRun
                 ? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
@@ -313,6 +314,7 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
                     .addClass('disabled').attr('disabled', true);
                 this.$el.find('.import-progress').show();
 
+                const runImport = () => {
                     Espo.Ajax.postRequest('DataMigration/action/import', {
                         backupName: this.selectedBackup,
                         preserveIds: preserveIds,
@@ -344,6 +346,40 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
                         this.$el.find('.import-progress').hide();
                         Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
                     });
+                };
+
+                if (createSnapshot) {
+                    Espo.Ui.notify(this.translate('Creating snapshot...', 'labels', 'DataMigration'));
+
+                    Espo.Ajax.postRequest('DataMigration/action/snapshot', {}, {timeout: 300000})
+                    .then(response => {
+                        if (response.success) {
+                            Espo.Ui.success(
+                                this.translate('Snapshot created', 'labels', 'DataMigration') +
+                                ': ' + response.snapshotName + ' (' + response.dumpSizeFormatted + ')'
+                            );
+                            runImport();
+                        } else {
+                            this.isImporting = false;
+                            this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                                .removeClass('disabled').attr('disabled', false);
+                            this.$el.find('.import-progress').hide();
+                            Espo.Ui.error(
+                                this.translate('Snapshot failed', 'labels', 'DataMigration') +
+                                ': ' + (response.error || 'Unknown error')
+                            );
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Snapshot failed', 'labels', 'DataMigration'));
+                    });
+                } else {
+                    runImport();
+                }
             });
         },
 

files/custom/Espo/Modules/DataMigration/Api/Snapshot.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace Espo\Modules\DataMigration\Api;
+
+use Espo\Core\Api\Action;
+use Espo\Core\Api\Request;
+use Espo\Core\Api\Response;
+use Espo\Core\Api\ResponseComposer;
+use Espo\Core\Exceptions\Forbidden;
+use Espo\Core\Utils\Config;
+use Espo\Entities\User;
+
+class Snapshot implements Action
+{
+    public function __construct(
+        private User $user,
+        private Config $config
+    ) {}
+
+    public function process(Request $request): Response
+    {
+        if (!$this->user->isAdmin()) {
+            throw new Forbidden();
+        }
+
+        $timestamp = date('Ymd-His');
+        $snapshotName = 'snapshot-' . $timestamp;
+        $snapshotDir = 'data/backups/' . $snapshotName;
+
+        mkdir($snapshotDir, 0755, true);
+
+        $dbParams = $this->config->get('database');
+        $host = $dbParams['host'] ?? 'localhost';
+        $port = $dbParams['port'] ?? '3306';
+        $dbName = $dbParams['dbname'] ?? '';
+        $user = $dbParams['user'] ?? '';
+        $password = $dbParams['password'] ?? '';
+
+        $dumpFile = $snapshotDir . '/database.sql';
+
+        $cmd = sprintf(
+            'mysqldump --host=%s --port=%s --user=%s --password=%s'
+            . ' --ssl=false --no-tablespaces --single-transaction --quick %s > %s 2>&1',
+            escapeshellarg($host),
+            escapeshellarg($port),
+            escapeshellarg($user),
+            escapeshellarg($password),
+            escapeshellarg($dbName),
+            escapeshellarg($dumpFile)
+        );
+
+        $output = [];
+        $returnCode = 0;
+        exec($cmd, $output, $returnCode);
+
+        if ($returnCode !== 0) {
+            // Cleanup on failure
+            if (file_exists($dumpFile)) {
+                unlink($dumpFile);
+            }
+
+            rmdir($snapshotDir);
+
+            return ResponseComposer::json([
+                'success' => false,
+                'error' => 'Database dump failed: ' . implode("\n", $output),
+            ]);
+        }
+
+        $dumpSize = filesize($dumpFile);
+
+        // Write a manifest
+        $manifest = [
+            'type' => 'snapshot',
+            'createdAt' => date('c'),
+            'sourceHost' => $this->config->get('siteUrl', 'unknown'),
+            'espoVersion' => $this->config->get('version', 'unknown'),
+            'database' => $dbName,
+            'dumpSizeBytes' => $dumpSize,
+        ];
+
+        file_put_contents(
+            $snapshotDir . '/manifest.json',
+            json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
+        );
+
+        return ResponseComposer::json([
+            'success' => true,
+            'snapshotName' => $snapshotName,
+            'dumpSize' => $dumpSize,
+            'dumpSizeFormatted' => $this->formatBytes($dumpSize),
+        ]);
+    }
+
+    private function formatBytes(int $bytes): string
+    {
+        if ($bytes >= 1073741824) {
+            return round($bytes / 1073741824, 2) . ' GB';
+        }
+
+        if ($bytes >= 1048576) {
+            return round($bytes / 1048576, 2) . ' MB';
+        }
+
+        if ($bytes >= 1024) {
+            return round($bytes / 1024, 2) . ' KB';
+        }
+
+        return $bytes . ' B';
+    }
+}

files/custom/Espo/Modules/DataMigration/Resources/i18n/en_US/DataMigration.json

@@ -25,6 +25,10 @@
         "Exporting...": "Exporting...",
         "Importing...": "Importing...",
         "Log": "Log",
+        "Create snapshot before import": "Create snapshot before import",
+        "Creating snapshot...": "Creating database snapshot...",
+        "Snapshot created": "Snapshot created",
+        "Snapshot failed": "Snapshot failed",
         "records": "records",
         "files": "files",
         "optional": "optional",
@@ -43,6 +47,7 @@
         "noFilesHint": "faster export, without attachment files",
         "preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
         "importNoFilesHint": "import only database records, skip attachment files",
+        "snapshotHint": "backup the entire database before importing, so you can restore if needed",
         "skipDuplicatesHint": "skip records that already exist in the database"
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/i18n/fa_IR/DataMigration.json

@@ -25,6 +25,10 @@
         "Exporting...": "מייצא...",
         "Importing...": "מייבא...",
         "Log": "לוג",
+        "Create snapshot before import": "צור צילום מצב לפני ייבוא",
+        "Creating snapshot...": "יוצר צילום מצב של מסד הנתונים...",
+        "Snapshot created": "צילום מצב נוצר",
+        "Snapshot failed": "יצירת צילום מצב נכשלה",
         "records": "רשומות",
         "files": "קבצים",
         "optional": "אופציונלי",
@@ -43,6 +47,7 @@
         "noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
         "preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
         "importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
+        "snapshotHint": "גיבוי מלא של מסד הנתונים לפני הייבוא, כדי שתוכל לשחזר במקרה הצורך",
         "skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/routes.json

@@ -23,5 +23,10 @@
         "route": "/DataMigration/action/deleteBackup",
         "method": "post",
         "actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
+    },
+    {
+        "route": "/DataMigration/action/snapshot",
+        "method": "post",
+        "actionClassName": "Espo\\Modules\\DataMigration\\Api\\Snapshot"
     }
 ]

files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php

@@ -309,11 +309,19 @@ class EntityImporter
         return $newId;
     }
 
+    /** @var array<string, string[]> Cache of existing column names per table */
+    private array $tableColumnsCache = [];
+
+    /** @var array<string, array<string, array>> Cache of column metadata per table */
+    private array $tableColumnMetaCache = [];
+
+    /** @var array<string, string[]> Skipped columns log per entity type */
+    private array $skippedColumnsLog = [];
+
     private function insertWithRawSql(string $entityType, array $record, string $newId): void
     {
-        $tableName = $this->entityManager
-            ->getQueryComposer()
-            ->toDb($entityType);
+        $tableName = $this->camelCaseToUnderscore($entityType);
+        $existingColumns = $this->getTableColumns($tableName);
 
         // Convert camelCase fields to snake_case for SQL
         $columns = [];
@@ -327,15 +335,25 @@ class EntityImporter
                 continue;
             }
 
-            // Skip non-storable array/object fields that come from getValueMap
-            if (is_array($value) || is_object($value)) {
+            $column = $this->camelCaseToUnderscore($field);
+
+            // Skip columns that don't exist in the target table
+            if (!in_array($column, $existingColumns)) {
+                if (!isset($this->skippedColumnsLog[$entityType][$column])) {
+                    $this->skippedColumnsLog[$entityType][$column] = true;
+                }
+
+                continue;
+            }
+
             // JSON fields should be stored as JSON strings
             if (is_array($value) || is_object($value)) {
                 $value = json_encode($value, JSON_UNESCAPED_UNICODE);
             }
-            }
 
-            $column = $this->camelCaseToUnderscore($field);
+            // Coerce value to match target column type
+            $value = $this->coerceValue($tableName, $column, $value);
+
             $columns[] = "`{$column}`";
             $values[] = $value;
             $placeholders[] = '?';
@@ -346,14 +364,85 @@ class EntityImporter
 
         $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
 
-        try {
         $pdo = $this->entityManager->getPDO();
         $stmt = $pdo->prepare($sql);
         $stmt->execute($values);
-        } catch (\Exception $e) {
-            // If column mismatch, try with just known columns
-            throw $e;
     }
+
+    private function getTableColumns(string $tableName): array
+    {
+        if (isset($this->tableColumnsCache[$tableName])) {
+            return $this->tableColumnsCache[$tableName];
+        }
+
+        $pdo = $this->entityManager->getPDO();
+        $sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
+        $columns = [];
+        $meta = [];
+
+        while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
+            $columns[] = $row['Field'];
+            $meta[$row['Field']] = [
+                'type' => $row['Type'],
+                'nullable' => $row['Null'] === 'YES',
+                'default' => $row['Default'],
+            ];
+        }
+
+        $this->tableColumnsCache[$tableName] = $columns;
+        $this->tableColumnMetaCache[$tableName] = $meta;
+
+        return $columns;
+    }
+
+    /**
+     * Coerce a value to be compatible with the target column type.
+     */
+    private function coerceValue(string $tableName, string $column, mixed $value): mixed
+    {
+        $meta = $this->tableColumnMetaCache[$tableName][$column] ?? null;
+
+        if ($meta === null) {
+            return $value;
+        }
+
+        $type = strtolower($meta['type']);
+        $isNumeric = str_contains($type, 'int') || str_contains($type, 'decimal')
+            || str_contains($type, 'float') || str_contains($type, 'double')
+            || str_contains($type, 'tinyint') || str_contains($type, 'smallint')
+            || str_contains($type, 'bigint');
+
+        if ($isNumeric && ($value === '' || $value === false)) {
+            if ($meta['nullable']) {
+                return null;
+            }
+
+            return 0;
+        }
+
+        if (!$meta['nullable'] && $value === null) {
+            if ($isNumeric) {
+                return 0;
+            }
+
+            return $meta['default'] ?? '';
+        }
+
+        return $value;
+    }
+
+    /**
+     * @return array<string, string[]> entityType => [skipped column names]
+     */
+    public function getSkippedColumns(): array
+    {
+        $result = [];
+
+        foreach ($this->skippedColumnsLog as $entityType => $columns) {
+            $result[$entityType] = array_keys($columns);
+        }
+
+        return $result;
     }
 
     private function resetAutoIncrement(string $entityType): void

files/custom/Espo/Modules/DataMigration/Services/ImportService.php

@@ -179,6 +179,18 @@ class ImportService
                 }
             }
         }
+
+        // Report skipped columns (missing in target DB)
+        $skippedColumns = $this->entityImporter->getSkippedColumns();
+
+        if (!empty($skippedColumns)) {
+            $io->writeLine('');
+            $io->writeLine('WARNING: Skipped columns not found in target database:');
+
+            foreach ($skippedColumns as $type => $columns) {
+                $io->writeLine("  {$type}: " . implode(', ', $columns));
+            }
+        }
     }
 
     private function entityTypeExists(string $entityType): bool

manifest.json

@@ -1,6 +1,6 @@
 {
     "name": "DataMigration",
-    "version": "1.1.4",
+    "version": "1.2.4",
     "acceptableVersions": [
         ">=8.0.0"
     ],