release: v1.3.2

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
fix: move Data Migration from separate 'customDev' section to 'customization'
2026-04-06 20:01:28 +00:00 · 2026-04-06 20:01:13 +00:00 · 2026-03-29 21:34:25 +00:00 · 2026-03-29 21:33:48 +00:00 · 2026-03-29 20:38:33 +00:00 · 2026-03-29 20:38:33 +00:00
12 changed files with 244 additions and 23 deletions
@@ -116,6 +116,19 @@
            </div>
        </div>

+        {{! === Snapshots === }}
+        <div class="panel panel-info">
+            <div class="panel-heading">
+                <h4 class="panel-title">
+                    <i class="fas fa-camera"></i>
+                    {{translate 'Database Snapshots' scope='DataMigration'}}
+                </h4>
+            </div>
+            <div class="panel-body" style="padding: 0;">
+                <div class="snapshots-list list-group" style="margin-bottom: 0;"></div>
+            </div>
+        </div>
+
        {{! === Import === }}
        <div class="import-section" style="display: none;">
            <div class="panel panel-warning">
@@ -72,6 +72,11 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
                const name = $(e.currentTarget).data('name');
                this.$el.find('.backup-details[data-name="' + name + '"]').toggle();
            },
+            'click [data-action="deleteSnapshot"]': function (e) {
+                e.stopPropagation();
+                const name = $(e.currentTarget).data('name');
+                this.actionDeleteSnapshot(name);
+            },
            'click [data-action="dismissExportLog"]': function () {
                this.exportLog = null;
                this.$el.find('.export-log-container').hide();
@@ -144,8 +149,10 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
            Espo.Ajax.getRequest('DataMigration/action/listBackups')
                .then(response => {
                    this.backups = response.backups || [];
+                    this.snapshots = response.snapshots || [];
                    this.isLoadingBackups = false;
                    this.renderBackups();
+                    this.renderSnapshots();
                })
                .catch(() => {
                    this.isLoadingBackups = false;
@@ -406,6 +413,60 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
            );
        },

+        renderSnapshots: function () {
+            const $container = this.$el.find('.snapshots-list');
+
+            if (!this.snapshots || this.snapshots.length === 0) {
+                $container.html(
+                    '<div class="text-muted" style="padding: 15px; text-align: center;">' +
+                    this.translate('No snapshots', 'labels', 'DataMigration') +
+                    '</div>'
+                );
+                return;
+            }
+
+            let html = '';
+
+            this.snapshots.forEach(snap => {
+                const date = snap.createdAt ? new Date(snap.createdAt).toLocaleString() : 'Unknown';
+
+                html += '<div class="list-group-item">';
+                html += '<div class="pull-right">';
+                html += '<button class="btn btn-danger btn-xs" data-action="deleteSnapshot" data-name="' + snap.name + '" title="' + this.translate('Delete') + '">';
+                html += '<i class="fas fa-trash"></i></button>';
+                html += '</div>';
+                html += '<h5 class="list-group-item-heading" style="margin: 0 0 4px;">';
+                html += '<i class="fas fa-camera text-info"></i> ' + snap.name;
+                html += '</h5>';
+                html += '<p class="list-group-item-text text-muted" style="margin: 0; font-size: 0.9em;">';
+                html += '<i class="fas fa-clock"></i> ' + date;
+                html += ' &nbsp;|&nbsp; <i class="fas fa-database"></i> ' + snap.database;
+                html += ' &nbsp;|&nbsp; <i class="fas fa-hdd"></i> ' + snap.sizeFormatted;
+                html += '</p>';
+                html += '</div>';
+            });
+
+            $container.html(html);
+        },
+
+        actionDeleteSnapshot: function (name) {
+            this.confirm(
+                this.translate('deleteSnapshotConfirmation', 'messages', 'DataMigration'),
+                () => {
+                    Espo.Ajax.postRequest('DataMigration/action/deleteBackup', {
+                        backupName: name,
+                    })
+                    .then(() => {
+                        Espo.Ui.success(this.translate('Deleted'));
+                        this.loadBackups();
+                    })
+                    .catch(() => {
+                        Espo.Ui.error(this.translate('Failed to delete snapshot', 'labels', 'DataMigration'));
+                    });
+                }
+            );
+        },
+
        renderLog: function (selector, lines, isSuccess) {
            const $container = this.$el.find(selector);

@@ -32,7 +32,7 @@ class DeleteBackup implements Action
        // Sanitize to prevent directory traversal
        $safeName = basename($backupName);

-        if (!str_starts_with($safeName, 'datamigration-')) {
+        if (!str_starts_with($safeName, 'datamigration-') && !str_starts_with($safeName, 'snapshot-')) {
            throw new Forbidden('Invalid backup name.');
        }

@@ -73,7 +73,43 @@ class ListBackups implements Action
            ];
        }

-        return ResponseComposer::json(['backups' => $backups]);
+        // Scan for snapshots
+        $snapshots = [];
+
+        foreach ($dirs as $dir) {
+            if ($dir === '.' || $dir === '..') {
+                continue;
+            }
+
+            if (!str_starts_with($dir, 'snapshot-')) {
+                continue;
+            }
+
+            $fullPath = $backupsDir . '/' . $dir;
+
+            if (!is_dir($fullPath)) {
+                continue;
+            }
+
+            $manifest = $this->manifestBuilder->read($fullPath);
+            $dumpFile = $fullPath . '/database.sql';
+            $dumpSize = file_exists($dumpFile) ? filesize($dumpFile) : 0;
+
+            $snapshots[] = [
+                'name' => $dir,
+                'createdAt' => $manifest['createdAt'] ?? null,
+                'sourceHost' => $manifest['sourceHost'] ?? 'unknown',
+                'espoVersion' => $manifest['espoVersion'] ?? 'unknown',
+                'database' => $manifest['database'] ?? 'unknown',
+                'dumpSizeBytes' => $dumpSize,
+                'sizeFormatted' => $this->formatBytes($dumpSize),
+            ];
+        }
+
+        return ResponseComposer::json([
+            'backups' => $backups,
+            'snapshots' => $snapshots,
+        ]);
    }

    private function formatBytes(int $bytes): string
@@ -39,7 +39,8 @@ class Snapshot implements Action
        $dumpFile = $snapshotDir . '/database.sql';

        $cmd = sprintf(
-            'mysqldump --host=%s --port=%s --user=%s --password=%s --single-transaction --quick %s > %s 2>&1',
+            'mysqldump --host=%s --port=%s --user=%s --password=%s'
+            . ' --ssl=false --no-tablespaces --single-transaction --quick %s > %s 2>&1',
            escapeshellarg($host),
            escapeshellarg($port),
            escapeshellarg($user),
@@ -16,6 +16,9 @@
        "Import failed": "Import failed",
        "Select a backup first": "Please select a backup first",
        "Failed to delete backup": "Failed to delete backup",
+        "Failed to delete snapshot": "Failed to delete snapshot",
+        "Database Snapshots": "Database Snapshots",
+        "No snapshots": "No snapshots found",
        "Selected backup": "Selected backup",
        "Import Options": "Import Options",
        "Preserve original IDs": "Preserve original IDs",
@@ -44,6 +47,7 @@
        "importConfirmation": "Are you sure you want to import this backup? This will add data to the database. This action cannot be easily undone.",
        "dryRunConfirmation": "Run a dry import? This will simulate the import without making any changes to the database.",
        "deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
+        "deleteSnapshotConfirmation": "Are you sure you want to delete this database snapshot? This action cannot be undone.",
        "noFilesHint": "faster export, without attachment files",
        "preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
        "importNoFilesHint": "import only database records, skip attachment files",
@@ -16,6 +16,9 @@
        "Import failed": "הייבוא נכשל",
        "Select a backup first": "נא לבחור גיבוי קודם",
        "Failed to delete backup": "מחיקת הגיבוי נכשלה",
+        "Failed to delete snapshot": "מחיקת צילום המצב נכשלה",
+        "Database Snapshots": "צילומי מצב של מסד הנתונים",
+        "No snapshots": "לא נמצאו צילומי מצב",
        "Selected backup": "גיבוי נבחר",
        "Import Options": "אפשרויות ייבוא",
        "Preserve original IDs": "שמור מזהים מקוריים",
@@ -44,6 +47,7 @@
        "importConfirmation": "האם אתה בטוח שברצונך לייבא גיבוי זה? פעולה זו תוסיף נתונים למסד הנתונים ולא ניתן לבטלה בקלות.",
        "dryRunConfirmation": "להריץ ייבוא ניסיון? פעולה זו תדמה את הייבוא מבלי לבצע שינויים במסד הנתונים.",
        "deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
+        "deleteSnapshotConfirmation": "האם אתה בטוח שברצונך למחוק צילום מצב זה? לא ניתן לבטל פעולה זו.",
        "noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
        "preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
        "importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
@@ -1,6 +1,5 @@
 {
-    "customDev": {
-        "label": "Custom Development",
+    "customization": {
        "itemList": [
            "__APPEND__",
            {
@@ -11,7 +10,6 @@
                "view": "modules/data-migration/views/admin/data-migration",
                "order": 50
            }
-        ],
-        "order": 30
+        ]
    }
 }
@@ -4,5 +4,6 @@
    "requires": {
        "espocrm": ">=8.0.0"
    },
-    "description": "Data migration tool for case data backup and restore"
+    "description": "Data migration tool for case data backup and restore",
+    "clientModule": "data-migration"
 }
@@ -309,9 +309,19 @@ class EntityImporter
        return $newId;
    }

+    /** @var array<string, string[]> Cache of existing column names per table */
+    private array $tableColumnsCache = [];
+
+    /** @var array<string, array<string, array>> Cache of column metadata per table */
+    private array $tableColumnMetaCache = [];
+
+    /** @var array<string, string[]> Skipped columns log per entity type */
+    private array $skippedColumnsLog = [];
+
    private function insertWithRawSql(string $entityType, array $record, string $newId): void
    {
        $tableName = $this->camelCaseToUnderscore($entityType);
+        $existingColumns = $this->getTableColumns($tableName);

        // Convert camelCase fields to snake_case for SQL
        $columns = [];
@@ -325,15 +335,25 @@ class EntityImporter
                continue;
            }

-            // Skip non-storable array/object fields that come from getValueMap
-            if (is_array($value) || is_object($value)) {
-                // JSON fields should be stored as JSON strings
-                if (is_array($value) || is_object($value)) {
-                    $value = json_encode($value, JSON_UNESCAPED_UNICODE);
+            $column = $this->camelCaseToUnderscore($field);
+
+            // Skip columns that don't exist in the target table
+            if (!in_array($column, $existingColumns)) {
+                if (!isset($this->skippedColumnsLog[$entityType][$column])) {
+                    $this->skippedColumnsLog[$entityType][$column] = true;
                }
+
+                continue;
            }

-            $column = $this->camelCaseToUnderscore($field);
+            // JSON fields should be stored as JSON strings
+            if (is_array($value) || is_object($value)) {
+                $value = json_encode($value, JSON_UNESCAPED_UNICODE);
+            }
+
+            // Coerce value to match target column type
+            $value = $this->coerceValue($tableName, $column, $value);
+
            $columns[] = "`{$column}`";
            $values[] = $value;
            $placeholders[] = '?';
@@ -344,14 +364,85 @@ class EntityImporter

        $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";

-        try {
-            $pdo = $this->entityManager->getPDO();
-            $stmt = $pdo->prepare($sql);
-            $stmt->execute($values);
-        } catch (\Exception $e) {
-            // If column mismatch, try with just known columns
-            throw $e;
+        $pdo = $this->entityManager->getPDO();
+        $stmt = $pdo->prepare($sql);
+        $stmt->execute($values);
+    }
+
+    private function getTableColumns(string $tableName): array
+    {
+        if (isset($this->tableColumnsCache[$tableName])) {
+            return $this->tableColumnsCache[$tableName];
        }
+
+        $pdo = $this->entityManager->getPDO();
+        $sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
+        $columns = [];
+        $meta = [];
+
+        while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
+            $columns[] = $row['Field'];
+            $meta[$row['Field']] = [
+                'type' => $row['Type'],
+                'nullable' => $row['Null'] === 'YES',
+                'default' => $row['Default'],
+            ];
+        }
+
+        $this->tableColumnsCache[$tableName] = $columns;
+        $this->tableColumnMetaCache[$tableName] = $meta;
+
+        return $columns;
+    }
+
+    /**
+     * Coerce a value to be compatible with the target column type.
+     */
+    private function coerceValue(string $tableName, string $column, mixed $value): mixed
+    {
+        $meta = $this->tableColumnMetaCache[$tableName][$column] ?? null;
+
+        if ($meta === null) {
+            return $value;
+        }
+
+        $type = strtolower($meta['type']);
+        $isNumeric = str_contains($type, 'int') || str_contains($type, 'decimal')
+            || str_contains($type, 'float') || str_contains($type, 'double')
+            || str_contains($type, 'tinyint') || str_contains($type, 'smallint')
+            || str_contains($type, 'bigint');
+
+        if ($isNumeric && ($value === '' || $value === false)) {
+            if ($meta['nullable']) {
+                return null;
+            }
+
+            return 0;
+        }
+
+        if (!$meta['nullable'] && $value === null) {
+            if ($isNumeric) {
+                return 0;
+            }
+
+            return $meta['default'] ?? '';
+        }
+
+        return $value;
+    }
+
+    /**
+     * @return array<string, string[]> entityType => [skipped column names]
+     */
+    public function getSkippedColumns(): array
+    {
+        $result = [];
+
+        foreach ($this->skippedColumnsLog as $entityType => $columns) {
+            $result[$entityType] = array_keys($columns);
+        }
+
+        return $result;
    }

    private function resetAutoIncrement(string $entityType): void
@@ -179,6 +179,18 @@ class ImportService
                }
            }
        }
+
+        // Report skipped columns (missing in target DB)
+        $skippedColumns = $this->entityImporter->getSkippedColumns();
+
+        if (!empty($skippedColumns)) {
+            $io->writeLine('');
+            $io->writeLine('WARNING: Skipped columns not found in target database:');
+
+            foreach ($skippedColumns as $type => $columns) {
+                $io->writeLine("  {$type}: " . implode(', ', $columns));
+            }
+        }
    }

    private function entityTypeExists(string $entityType): bool
@@ -1,13 +1,13 @@
 {
    "name": "DataMigration",
-    "version": "1.2.0",
+    "version": "1.3.2",
    "acceptableVersions": [
        ">=8.0.0"
    ],
    "php": [
        ">=8.1"
    ],
-    "releaseDate": "2026-03-29",
+    "releaseDate": "2026-04-06",
    "author": "klear",
    "description": "Case data migration: export/import all case-related data between EspoCRM instances"
 }
Author	SHA1	Message	Date
chaim	258a5b8136	release: v1.3.2 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-06 20:01:28 +00:00
chaim	3dc5c4e571	fix: move Data Migration from separate 'customDev' section to 'customization' Avoid creating separate admin panel sections for each extension. All custom settings go under the existing "התאמה אישית" section. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-06 20:01:13 +00:00
chaim	8b745bc99f	release: v1.3.1	2026-03-29 21:34:25 +00:00
chaim	384d7c6df0	fix: add clientModule to module.json for client-side template resolution Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-03-29 21:33:48 +00:00
chaim	49275d0fc9	release: v1.3.0	2026-03-29 20:38:33 +00:00
chaim	ec759b89c1	feat: snapshot management UI — list and delete database snapshots - ListBackups API now returns snapshots alongside backups - DeleteBackup API accepts snapshot- prefixed names - New "Database Snapshots" panel in admin UI with delete buttons - i18n: English + Hebrew translations Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-03-29 20:38:33 +00:00
chaim	d5fe6c21a0	release: v1.2.4	2026-03-29 20:31:11 +00:00
chaim	54e8c77325	fix: proper type coercion for raw SQL import columns Read column metadata (type, nullable, default) from SHOW COLUMNS and coerce values accordingly: empty string/false → 0 for NOT NULL numeric columns, null for nullable ones. Fixes is_internal import error. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-03-29 20:31:11 +00:00
chaim	35cda6e91c	release: v1.2.3	2026-03-29 20:28:56 +00:00
chaim	ba00e7ecf6	fix: convert empty strings to null for integer/bool columns MySQL strict mode rejects empty string '' for integer columns like is_internal. Convert '' and false to null before SQL insert. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-03-29 20:28:56 +00:00
chaim	e3a1be1844	release: v1.2.2	2026-03-29 20:25:54 +00:00
chaim	fd279b6c8b	fix: skip columns missing in target DB during raw SQL import When importing entities with autoincrement (Case, Charge, Invoice), validate each column exists in the target table before inserting. Columns from modules not installed on the target are silently skipped and reported in the import log summary. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-03-29 20:25:54 +00:00
chaim	73b5907539	release: v1.2.1	2026-03-29 20:22:05 +00:00
chaim	edeef544d4	fix: mysqldump SSL and tablespace errors in snapshot Add --ssl=false and --no-tablespaces flags to handle Docker MySQL connections with self-signed certificates. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-03-29 20:22:05 +00:00