release: v1.2.4

Read column metadata (type, nullable, default) from SHOW COLUMNS and
coerce values accordingly: empty string/false → 0 for NOT NULL numeric
columns, null for nullable ones. Fixes is_internal import error.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

files/client/custom/modules/data-migration/res/templates/admin/data-migration.tpl

@@ -150,6 +150,22 @@
                                 <span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
                             </label>
                         </div>
+
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="importNoFiles">
+                                {{translate 'Skip attachment files' scope='DataMigration'}}
+                                <span class="text-muted"> — {{translate 'importNoFilesHint' category='messages' scope='DataMigration'}}</span>
+                            </label>
+                        </div>
+
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="createSnapshot" checked>
+                                {{translate 'Create snapshot before import' scope='DataMigration'}}
+                                <span class="text-muted"> — {{translate 'snapshotHint' category='messages' scope='DataMigration'}}</span>
+                            </label>
+                        </div>
                     </div>
 
                     <div class="btn-group">

files/client/custom/modules/data-migration/src/views/admin/data-migration.js

@@ -301,6 +301,8 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
 
             const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
             const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
+            const noFiles = this.$el.find('input[name="importNoFiles"]').is(':checked');
+            const createSnapshot = !dryRun && this.$el.find('input[name="createSnapshot"]').is(':checked');
 
             const confirmMsg = dryRun
                 ? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
@@ -312,36 +314,72 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
                     .addClass('disabled').attr('disabled', true);
                 this.$el.find('.import-progress').show();
 
-                Espo.Ajax.postRequest('DataMigration/action/import', {
-                    backupName: this.selectedBackup,
-                    preserveIds: preserveIds,
-                    skipDuplicates: skipDuplicates,
-                    dryRun: dryRun,
-                }, {timeout: 600000})
-                .then(response => {
-                    this.isImporting = false;
-                    this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
-                        .removeClass('disabled').attr('disabled', false);
-                    this.$el.find('.import-progress').hide();
+                const runImport = () => {
+                    Espo.Ajax.postRequest('DataMigration/action/import', {
+                        backupName: this.selectedBackup,
+                        preserveIds: preserveIds,
+                        noFiles: noFiles,
+                        skipDuplicates: skipDuplicates,
+                        dryRun: dryRun,
+                    }, {timeout: 600000})
+                    .then(response => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
 
-                    this.importLog = response.log || [];
-                    this.renderLog('.import-log-container', this.importLog, !dryRun);
+                        this.importLog = response.log || [];
+                        this.renderLog('.import-log-container', this.importLog, !dryRun);
 
-                    const label = dryRun ? 'Dry run completed' : 'Import completed';
+                        const label = dryRun ? 'Dry run completed' : 'Import completed';
 
-                    Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
+                        Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
 
-                    if (!dryRun) {
-                        this.loadStatus();
-                    }
-                })
-                .catch(xhr => {
-                    this.isImporting = false;
-                    this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
-                        .removeClass('disabled').attr('disabled', false);
-                    this.$el.find('.import-progress').hide();
-                    Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
-                });
+                        if (!dryRun) {
+                            this.loadStatus();
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
+                    });
+                };
+
+                if (createSnapshot) {
+                    Espo.Ui.notify(this.translate('Creating snapshot...', 'labels', 'DataMigration'));
+
+                    Espo.Ajax.postRequest('DataMigration/action/snapshot', {}, {timeout: 300000})
+                    .then(response => {
+                        if (response.success) {
+                            Espo.Ui.success(
+                                this.translate('Snapshot created', 'labels', 'DataMigration') +
+                                ': ' + response.snapshotName + ' (' + response.dumpSizeFormatted + ')'
+                            );
+                            runImport();
+                        } else {
+                            this.isImporting = false;
+                            this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                                .removeClass('disabled').attr('disabled', false);
+                            this.$el.find('.import-progress').hide();
+                            Espo.Ui.error(
+                                this.translate('Snapshot failed', 'labels', 'DataMigration') +
+                                ': ' + (response.error || 'Unknown error')
+                            );
+                        }
+                    })
+                    .catch(xhr => {
+                        this.isImporting = false;
+                        this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
+                            .removeClass('disabled').attr('disabled', false);
+                        this.$el.find('.import-progress').hide();
+                        Espo.Ui.error(this.translate('Snapshot failed', 'labels', 'DataMigration'));
+                    });
+                } else {
+                    runImport();
+                }
             });
         },
 

files/custom/Espo/Modules/DataMigration/Api/Import.php

@@ -46,6 +46,7 @@ class Import implements Action
         $preserveIds = $body->preserveIds ?? true;
         $skipDuplicates = $body->skipDuplicates ?? true;
         $dryRun = $body->dryRun ?? false;
+        $noFiles = $body->noFiles ?? false;
 
         $io = new BufferedIO();
 
@@ -54,6 +55,7 @@ class Import implements Action
             'preserveIds' => $preserveIds,
             'skipDuplicates' => $skipDuplicates,
             'dryRun' => $dryRun,
+            'noFiles' => $noFiles,
             'verbose' => true,
         ], $io);
 

files/custom/Espo/Modules/DataMigration/Api/Snapshot.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace Espo\Modules\DataMigration\Api;
+
+use Espo\Core\Api\Action;
+use Espo\Core\Api\Request;
+use Espo\Core\Api\Response;
+use Espo\Core\Api\ResponseComposer;
+use Espo\Core\Exceptions\Forbidden;
+use Espo\Core\Utils\Config;
+use Espo\Entities\User;
+
+class Snapshot implements Action
+{
+    public function __construct(
+        private User $user,
+        private Config $config
+    ) {}
+
+    public function process(Request $request): Response
+    {
+        if (!$this->user->isAdmin()) {
+            throw new Forbidden();
+        }
+
+        $timestamp = date('Ymd-His');
+        $snapshotName = 'snapshot-' . $timestamp;
+        $snapshotDir = 'data/backups/' . $snapshotName;
+
+        mkdir($snapshotDir, 0755, true);
+
+        $dbParams = $this->config->get('database');
+        $host = $dbParams['host'] ?? 'localhost';
+        $port = $dbParams['port'] ?? '3306';
+        $dbName = $dbParams['dbname'] ?? '';
+        $user = $dbParams['user'] ?? '';
+        $password = $dbParams['password'] ?? '';
+
+        $dumpFile = $snapshotDir . '/database.sql';
+
+        $cmd = sprintf(
+            'mysqldump --host=%s --port=%s --user=%s --password=%s'
+            . ' --ssl=false --no-tablespaces --single-transaction --quick %s > %s 2>&1',
+            escapeshellarg($host),
+            escapeshellarg($port),
+            escapeshellarg($user),
+            escapeshellarg($password),
+            escapeshellarg($dbName),
+            escapeshellarg($dumpFile)
+        );
+
+        $output = [];
+        $returnCode = 0;
+        exec($cmd, $output, $returnCode);
+
+        if ($returnCode !== 0) {
+            // Cleanup on failure
+            if (file_exists($dumpFile)) {
+                unlink($dumpFile);
+            }
+
+            rmdir($snapshotDir);
+
+            return ResponseComposer::json([
+                'success' => false,
+                'error' => 'Database dump failed: ' . implode("\n", $output),
+            ]);
+        }
+
+        $dumpSize = filesize($dumpFile);
+
+        // Write a manifest
+        $manifest = [
+            'type' => 'snapshot',
+            'createdAt' => date('c'),
+            'sourceHost' => $this->config->get('siteUrl', 'unknown'),
+            'espoVersion' => $this->config->get('version', 'unknown'),
+            'database' => $dbName,
+            'dumpSizeBytes' => $dumpSize,
+        ];
+
+        file_put_contents(
+            $snapshotDir . '/manifest.json',
+            json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
+        );
+
+        return ResponseComposer::json([
+            'success' => true,
+            'snapshotName' => $snapshotName,
+            'dumpSize' => $dumpSize,
+            'dumpSizeFormatted' => $this->formatBytes($dumpSize),
+        ]);
+    }
+
+    private function formatBytes(int $bytes): string
+    {
+        if ($bytes >= 1073741824) {
+            return round($bytes / 1073741824, 2) . ' GB';
+        }
+
+        if ($bytes >= 1048576) {
+            return round($bytes / 1048576, 2) . ' MB';
+        }
+
+        if ($bytes >= 1024) {
+            return round($bytes / 1024, 2) . ' KB';
+        }
+
+        return $bytes . ' B';
+    }
+}

files/custom/Espo/Modules/DataMigration/Resources/i18n/en_US/DataMigration.json

@@ -25,6 +25,10 @@
         "Exporting...": "Exporting...",
         "Importing...": "Importing...",
         "Log": "Log",
+        "Create snapshot before import": "Create snapshot before import",
+        "Creating snapshot...": "Creating database snapshot...",
+        "Snapshot created": "Snapshot created",
+        "Snapshot failed": "Snapshot failed",
         "records": "records",
         "files": "files",
         "optional": "optional",
@@ -42,6 +46,8 @@
         "deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
         "noFilesHint": "faster export, without attachment files",
         "preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
+        "importNoFilesHint": "import only database records, skip attachment files",
+        "snapshotHint": "backup the entire database before importing, so you can restore if needed",
         "skipDuplicatesHint": "skip records that already exist in the database"
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/i18n/fa_IR/Admin.json

@@ -1,6 +1,7 @@
 {
     "labels": {
-        "Data Migration": "העברת נתונים"
+        "Data Migration": "העברת נתונים",
+        "Custom Development": "פיתוח עצמאי"
     },
     "descriptions": {
         "dataMigration": "ייצוא וייבוא נתוני תיקים בין מערכות EspoCRM"

files/custom/Espo/Modules/DataMigration/Resources/i18n/fa_IR/DataMigration.json

@@ -25,6 +25,10 @@
         "Exporting...": "מייצא...",
         "Importing...": "מייבא...",
         "Log": "לוג",
+        "Create snapshot before import": "צור צילום מצב לפני ייבוא",
+        "Creating snapshot...": "יוצר צילום מצב של מסד הנתונים...",
+        "Snapshot created": "צילום מצב נוצר",
+        "Snapshot failed": "יצירת צילום מצב נכשלה",
         "records": "רשומות",
         "files": "קבצים",
         "optional": "אופציונלי",
@@ -42,6 +46,8 @@
         "deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
         "noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
         "preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
+        "importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
+        "snapshotHint": "גיבוי מלא של מסד הנתונים לפני הייבוא, כדי שתוכל לשחזר במקרה הצורך",
         "skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/metadata/app/adminPanel.json

@@ -1,14 +1,17 @@
 {
-    "system": {
+    "customDev": {
+        "label": "Custom Development",
         "itemList": [
+            "__APPEND__",
             {
                 "url": "#Admin/dataMigration",
                 "label": "Data Migration",
                 "iconClass": "fas fa-exchange-alt",
                 "description": "dataMigration",
                 "view": "modules/data-migration/views/admin/data-migration",
-                "order": 85
+                "order": 50
             }
-        ]
+        ],
+        "order": 30
     }
 }

files/custom/Espo/Modules/DataMigration/Resources/routes.json

@@ -23,5 +23,10 @@
         "route": "/DataMigration/action/deleteBackup",
         "method": "post",
         "actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
+    },
+    {
+        "route": "/DataMigration/action/snapshot",
+        "method": "post",
+        "actionClassName": "Espo\\Modules\\DataMigration\\Api\\Snapshot"
     }
 ]

files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php

@@ -309,11 +309,19 @@ class EntityImporter
         return $newId;
     }
 
+    /** @var array<string, string[]> Cache of existing column names per table */
+    private array $tableColumnsCache = [];
+
+    /** @var array<string, array<string, array>> Cache of column metadata per table */
+    private array $tableColumnMetaCache = [];
+
+    /** @var array<string, string[]> Skipped columns log per entity type */
+    private array $skippedColumnsLog = [];
+
     private function insertWithRawSql(string $entityType, array $record, string $newId): void
     {
-        $tableName = $this->entityManager
-            ->getQueryComposer()
-            ->toDb($entityType);
+        $tableName = $this->camelCaseToUnderscore($entityType);
+        $existingColumns = $this->getTableColumns($tableName);
 
         // Convert camelCase fields to snake_case for SQL
         $columns = [];
@@ -327,15 +335,25 @@ class EntityImporter
                 continue;
             }
 
-            // Skip non-storable array/object fields that come from getValueMap
-            if (is_array($value) || is_object($value)) {
-                // JSON fields should be stored as JSON strings
-                if (is_array($value) || is_object($value)) {
-                    $value = json_encode($value, JSON_UNESCAPED_UNICODE);
+            $column = $this->camelCaseToUnderscore($field);
+
+            // Skip columns that don't exist in the target table
+            if (!in_array($column, $existingColumns)) {
+                if (!isset($this->skippedColumnsLog[$entityType][$column])) {
+                    $this->skippedColumnsLog[$entityType][$column] = true;
                 }
+
+                continue;
             }
 
-            $column = $this->camelCaseToUnderscore($field);
+            // JSON fields should be stored as JSON strings
+            if (is_array($value) || is_object($value)) {
+                $value = json_encode($value, JSON_UNESCAPED_UNICODE);
+            }
+
+            // Coerce value to match target column type
+            $value = $this->coerceValue($tableName, $column, $value);
+
             $columns[] = "`{$column}`";
             $values[] = $value;
             $placeholders[] = '?';
@@ -346,14 +364,85 @@ class EntityImporter
 
         $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
 
-        try {
-            $pdo = $this->entityManager->getPDO();
-            $stmt = $pdo->prepare($sql);
-            $stmt->execute($values);
-        } catch (\Exception $e) {
-            // If column mismatch, try with just known columns
-            throw $e;
+        $pdo = $this->entityManager->getPDO();
+        $stmt = $pdo->prepare($sql);
+        $stmt->execute($values);
+    }
+
+    private function getTableColumns(string $tableName): array
+    {
+        if (isset($this->tableColumnsCache[$tableName])) {
+            return $this->tableColumnsCache[$tableName];
         }
+
+        $pdo = $this->entityManager->getPDO();
+        $sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
+        $columns = [];
+        $meta = [];
+
+        while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
+            $columns[] = $row['Field'];
+            $meta[$row['Field']] = [
+                'type' => $row['Type'],
+                'nullable' => $row['Null'] === 'YES',
+                'default' => $row['Default'],
+            ];
+        }
+
+        $this->tableColumnsCache[$tableName] = $columns;
+        $this->tableColumnMetaCache[$tableName] = $meta;
+
+        return $columns;
+    }
+
+    /**
+     * Coerce a value to be compatible with the target column type.
+     */
+    private function coerceValue(string $tableName, string $column, mixed $value): mixed
+    {
+        $meta = $this->tableColumnMetaCache[$tableName][$column] ?? null;
+
+        if ($meta === null) {
+            return $value;
+        }
+
+        $type = strtolower($meta['type']);
+        $isNumeric = str_contains($type, 'int') || str_contains($type, 'decimal')
+            || str_contains($type, 'float') || str_contains($type, 'double')
+            || str_contains($type, 'tinyint') || str_contains($type, 'smallint')
+            || str_contains($type, 'bigint');
+
+        if ($isNumeric && ($value === '' || $value === false)) {
+            if ($meta['nullable']) {
+                return null;
+            }
+
+            return 0;
+        }
+
+        if (!$meta['nullable'] && $value === null) {
+            if ($isNumeric) {
+                return 0;
+            }
+
+            return $meta['default'] ?? '';
+        }
+
+        return $value;
+    }
+
+    /**
+     * @return array<string, string[]> entityType => [skipped column names]
+     */
+    public function getSkippedColumns(): array
+    {
+        $result = [];
+
+        foreach ($this->skippedColumnsLog as $entityType => $columns) {
+            $result[$entityType] = array_keys($columns);
+        }
+
+        return $result;
     }
 
     private function resetAutoIncrement(string $entityType): void

files/custom/Espo/Modules/DataMigration/Services/ImportService.php

@@ -62,6 +62,7 @@ class ImportService
         $preserveIds = $options['preserveIds'] ?? true;
         $skipDuplicates = $options['skipDuplicates'] ?? false;
         $dryRun = $options['dryRun'] ?? false;
+        $noFiles = $options['noFiles'] ?? false;
         $verbose = $options['verbose'] ?? false;
 
         $dataDir = rtrim($inputDir, '/') . '/data';
@@ -100,6 +101,11 @@ class ImportService
                 continue;
             }
 
+            if (!$this->entityTypeExists($entityType)) {
+                $io->writeLine("Skipping {$entityType} (module not installed)");
+                continue;
+            }
+
             $io->writeLine("Importing {$entityType}...");
 
             $result = $this->entityImporter->import(
@@ -133,7 +139,10 @@ class ImportService
         }
 
         // Restore attachment files
-        if (is_dir($filesDir)) {
+        if ($noFiles) {
+            $io->writeLine('');
+            $io->writeLine('Attachment files: SKIPPED (--no-files)');
+        } else if (is_dir($filesDir)) {
             $io->writeLine('');
             $io->writeLine('Restoring attachment files...');
 
@@ -170,5 +179,27 @@ class ImportService
                 }
             }
         }
+
+        // Report skipped columns (missing in target DB)
+        $skippedColumns = $this->entityImporter->getSkippedColumns();
+
+        if (!empty($skippedColumns)) {
+            $io->writeLine('');
+            $io->writeLine('WARNING: Skipped columns not found in target database:');
+
+            foreach ($skippedColumns as $type => $columns) {
+                $io->writeLine("  {$type}: " . implode(', ', $columns));
+            }
+        }
+    }
+
+    private function entityTypeExists(string $entityType): bool
+    {
+        try {
+            $this->entityManager->getRDBRepository($entityType);
+            return true;
+        } catch (\Exception $e) {
+            return false;
+        }
     }
 }

manifest.json

@@ -1,6 +1,6 @@
 {
     "name": "DataMigration",
-    "version": "1.1.0",
+    "version": "1.2.4",
     "acceptableVersions": [
         ">=8.0.0"
     ],