Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 258a5b8136 | |||
| 3dc5c4e571 | |||
| 8b745bc99f | |||
| 384d7c6df0 | |||
| 49275d0fc9 | |||
| ec759b89c1 | |||
| d5fe6c21a0 | |||
| 54e8c77325 | |||
| 35cda6e91c | |||
| ba00e7ecf6 | |||
| e3a1be1844 | |||
| fd279b6c8b | |||
| 73b5907539 | |||
| edeef544d4 | |||
| 1964ad99b6 | |||
| 078b2e6051 | |||
| a57e23dec3 | |||
| a8c4a2ebe1 |
@@ -116,6 +116,19 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{! === Snapshots === }}
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<h4 class="panel-title">
|
||||
<i class="fas fa-camera"></i>
|
||||
{{translate 'Database Snapshots' scope='DataMigration'}}
|
||||
</h4>
|
||||
</div>
|
||||
<div class="panel-body" style="padding: 0;">
|
||||
<div class="snapshots-list list-group" style="margin-bottom: 0;"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{! === Import === }}
|
||||
<div class="import-section" style="display: none;">
|
||||
<div class="panel panel-warning">
|
||||
@@ -150,6 +163,22 @@
|
||||
<span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="importNoFiles">
|
||||
{{translate 'Skip attachment files' scope='DataMigration'}}
|
||||
<span class="text-muted"> — {{translate 'importNoFilesHint' category='messages' scope='DataMigration'}}</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="createSnapshot" checked>
|
||||
{{translate 'Create snapshot before import' scope='DataMigration'}}
|
||||
<span class="text-muted"> — {{translate 'snapshotHint' category='messages' scope='DataMigration'}}</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="btn-group">
|
||||
|
||||
@@ -72,6 +72,11 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
||||
const name = $(e.currentTarget).data('name');
|
||||
this.$el.find('.backup-details[data-name="' + name + '"]').toggle();
|
||||
},
|
||||
'click [data-action="deleteSnapshot"]': function (e) {
|
||||
e.stopPropagation();
|
||||
const name = $(e.currentTarget).data('name');
|
||||
this.actionDeleteSnapshot(name);
|
||||
},
|
||||
'click [data-action="dismissExportLog"]': function () {
|
||||
this.exportLog = null;
|
||||
this.$el.find('.export-log-container').hide();
|
||||
@@ -144,8 +149,10 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
||||
Espo.Ajax.getRequest('DataMigration/action/listBackups')
|
||||
.then(response => {
|
||||
this.backups = response.backups || [];
|
||||
this.snapshots = response.snapshots || [];
|
||||
this.isLoadingBackups = false;
|
||||
this.renderBackups();
|
||||
this.renderSnapshots();
|
||||
})
|
||||
.catch(() => {
|
||||
this.isLoadingBackups = false;
|
||||
@@ -301,6 +308,8 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
||||
|
||||
const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
|
||||
const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
|
||||
const noFiles = this.$el.find('input[name="importNoFiles"]').is(':checked');
|
||||
const createSnapshot = !dryRun && this.$el.find('input[name="createSnapshot"]').is(':checked');
|
||||
|
||||
const confirmMsg = dryRun
|
||||
? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
|
||||
@@ -312,36 +321,72 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
||||
.addClass('disabled').attr('disabled', true);
|
||||
this.$el.find('.import-progress').show();
|
||||
|
||||
Espo.Ajax.postRequest('DataMigration/action/import', {
|
||||
backupName: this.selectedBackup,
|
||||
preserveIds: preserveIds,
|
||||
skipDuplicates: skipDuplicates,
|
||||
dryRun: dryRun,
|
||||
}, {timeout: 600000})
|
||||
.then(response => {
|
||||
this.isImporting = false;
|
||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||
.removeClass('disabled').attr('disabled', false);
|
||||
this.$el.find('.import-progress').hide();
|
||||
const runImport = () => {
|
||||
Espo.Ajax.postRequest('DataMigration/action/import', {
|
||||
backupName: this.selectedBackup,
|
||||
preserveIds: preserveIds,
|
||||
noFiles: noFiles,
|
||||
skipDuplicates: skipDuplicates,
|
||||
dryRun: dryRun,
|
||||
}, {timeout: 600000})
|
||||
.then(response => {
|
||||
this.isImporting = false;
|
||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||
.removeClass('disabled').attr('disabled', false);
|
||||
this.$el.find('.import-progress').hide();
|
||||
|
||||
this.importLog = response.log || [];
|
||||
this.renderLog('.import-log-container', this.importLog, !dryRun);
|
||||
this.importLog = response.log || [];
|
||||
this.renderLog('.import-log-container', this.importLog, !dryRun);
|
||||
|
||||
const label = dryRun ? 'Dry run completed' : 'Import completed';
|
||||
const label = dryRun ? 'Dry run completed' : 'Import completed';
|
||||
|
||||
Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
|
||||
Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
|
||||
|
||||
if (!dryRun) {
|
||||
this.loadStatus();
|
||||
}
|
||||
})
|
||||
.catch(xhr => {
|
||||
this.isImporting = false;
|
||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||
.removeClass('disabled').attr('disabled', false);
|
||||
this.$el.find('.import-progress').hide();
|
||||
Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
|
||||
});
|
||||
if (!dryRun) {
|
||||
this.loadStatus();
|
||||
}
|
||||
})
|
||||
.catch(xhr => {
|
||||
this.isImporting = false;
|
||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||
.removeClass('disabled').attr('disabled', false);
|
||||
this.$el.find('.import-progress').hide();
|
||||
Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
|
||||
});
|
||||
};
|
||||
|
||||
if (createSnapshot) {
|
||||
Espo.Ui.notify(this.translate('Creating snapshot...', 'labels', 'DataMigration'));
|
||||
|
||||
Espo.Ajax.postRequest('DataMigration/action/snapshot', {}, {timeout: 300000})
|
||||
.then(response => {
|
||||
if (response.success) {
|
||||
Espo.Ui.success(
|
||||
this.translate('Snapshot created', 'labels', 'DataMigration') +
|
||||
': ' + response.snapshotName + ' (' + response.dumpSizeFormatted + ')'
|
||||
);
|
||||
runImport();
|
||||
} else {
|
||||
this.isImporting = false;
|
||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||
.removeClass('disabled').attr('disabled', false);
|
||||
this.$el.find('.import-progress').hide();
|
||||
Espo.Ui.error(
|
||||
this.translate('Snapshot failed', 'labels', 'DataMigration') +
|
||||
': ' + (response.error || 'Unknown error')
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch(xhr => {
|
||||
this.isImporting = false;
|
||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||
.removeClass('disabled').attr('disabled', false);
|
||||
this.$el.find('.import-progress').hide();
|
||||
Espo.Ui.error(this.translate('Snapshot failed', 'labels', 'DataMigration'));
|
||||
});
|
||||
} else {
|
||||
runImport();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
@@ -368,6 +413,60 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
||||
);
|
||||
},
|
||||
|
||||
renderSnapshots: function () {
|
||||
const $container = this.$el.find('.snapshots-list');
|
||||
|
||||
if (!this.snapshots || this.snapshots.length === 0) {
|
||||
$container.html(
|
||||
'<div class="text-muted" style="padding: 15px; text-align: center;">' +
|
||||
this.translate('No snapshots', 'labels', 'DataMigration') +
|
||||
'</div>'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let html = '';
|
||||
|
||||
this.snapshots.forEach(snap => {
|
||||
const date = snap.createdAt ? new Date(snap.createdAt).toLocaleString() : 'Unknown';
|
||||
|
||||
html += '<div class="list-group-item">';
|
||||
html += '<div class="pull-right">';
|
||||
html += '<button class="btn btn-danger btn-xs" data-action="deleteSnapshot" data-name="' + snap.name + '" title="' + this.translate('Delete') + '">';
|
||||
html += '<i class="fas fa-trash"></i></button>';
|
||||
html += '</div>';
|
||||
html += '<h5 class="list-group-item-heading" style="margin: 0 0 4px;">';
|
||||
html += '<i class="fas fa-camera text-info"></i> ' + snap.name;
|
||||
html += '</h5>';
|
||||
html += '<p class="list-group-item-text text-muted" style="margin: 0; font-size: 0.9em;">';
|
||||
html += '<i class="fas fa-clock"></i> ' + date;
|
||||
html += ' | <i class="fas fa-database"></i> ' + snap.database;
|
||||
html += ' | <i class="fas fa-hdd"></i> ' + snap.sizeFormatted;
|
||||
html += '</p>';
|
||||
html += '</div>';
|
||||
});
|
||||
|
||||
$container.html(html);
|
||||
},
|
||||
|
||||
actionDeleteSnapshot: function (name) {
|
||||
this.confirm(
|
||||
this.translate('deleteSnapshotConfirmation', 'messages', 'DataMigration'),
|
||||
() => {
|
||||
Espo.Ajax.postRequest('DataMigration/action/deleteBackup', {
|
||||
backupName: name,
|
||||
})
|
||||
.then(() => {
|
||||
Espo.Ui.success(this.translate('Deleted'));
|
||||
this.loadBackups();
|
||||
})
|
||||
.catch(() => {
|
||||
Espo.Ui.error(this.translate('Failed to delete snapshot', 'labels', 'DataMigration'));
|
||||
});
|
||||
}
|
||||
);
|
||||
},
|
||||
|
||||
renderLog: function (selector, lines, isSuccess) {
|
||||
const $container = this.$el.find(selector);
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class DeleteBackup implements Action
|
||||
// Sanitize to prevent directory traversal
|
||||
$safeName = basename($backupName);
|
||||
|
||||
if (!str_starts_with($safeName, 'datamigration-')) {
|
||||
if (!str_starts_with($safeName, 'datamigration-') && !str_starts_with($safeName, 'snapshot-')) {
|
||||
throw new Forbidden('Invalid backup name.');
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ class Import implements Action
|
||||
$preserveIds = $body->preserveIds ?? true;
|
||||
$skipDuplicates = $body->skipDuplicates ?? true;
|
||||
$dryRun = $body->dryRun ?? false;
|
||||
$noFiles = $body->noFiles ?? false;
|
||||
|
||||
$io = new BufferedIO();
|
||||
|
||||
@@ -54,6 +55,7 @@ class Import implements Action
|
||||
'preserveIds' => $preserveIds,
|
||||
'skipDuplicates' => $skipDuplicates,
|
||||
'dryRun' => $dryRun,
|
||||
'noFiles' => $noFiles,
|
||||
'verbose' => true,
|
||||
], $io);
|
||||
|
||||
|
||||
@@ -73,7 +73,43 @@ class ListBackups implements Action
|
||||
];
|
||||
}
|
||||
|
||||
return ResponseComposer::json(['backups' => $backups]);
|
||||
// Scan for snapshots
|
||||
$snapshots = [];
|
||||
|
||||
foreach ($dirs as $dir) {
|
||||
if ($dir === '.' || $dir === '..') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!str_starts_with($dir, 'snapshot-')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$fullPath = $backupsDir . '/' . $dir;
|
||||
|
||||
if (!is_dir($fullPath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$manifest = $this->manifestBuilder->read($fullPath);
|
||||
$dumpFile = $fullPath . '/database.sql';
|
||||
$dumpSize = file_exists($dumpFile) ? filesize($dumpFile) : 0;
|
||||
|
||||
$snapshots[] = [
|
||||
'name' => $dir,
|
||||
'createdAt' => $manifest['createdAt'] ?? null,
|
||||
'sourceHost' => $manifest['sourceHost'] ?? 'unknown',
|
||||
'espoVersion' => $manifest['espoVersion'] ?? 'unknown',
|
||||
'database' => $manifest['database'] ?? 'unknown',
|
||||
'dumpSizeBytes' => $dumpSize,
|
||||
'sizeFormatted' => $this->formatBytes($dumpSize),
|
||||
];
|
||||
}
|
||||
|
||||
return ResponseComposer::json([
|
||||
'backups' => $backups,
|
||||
'snapshots' => $snapshots,
|
||||
]);
|
||||
}
|
||||
|
||||
private function formatBytes(int $bytes): string
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Api;
|
||||
|
||||
use Espo\Core\Api\Action;
|
||||
use Espo\Core\Api\Request;
|
||||
use Espo\Core\Api\Response;
|
||||
use Espo\Core\Api\ResponseComposer;
|
||||
use Espo\Core\Exceptions\Forbidden;
|
||||
use Espo\Core\Utils\Config;
|
||||
use Espo\Entities\User;
|
||||
|
||||
class Snapshot implements Action
|
||||
{
|
||||
public function __construct(
|
||||
private User $user,
|
||||
private Config $config
|
||||
) {}
|
||||
|
||||
public function process(Request $request): Response
|
||||
{
|
||||
if (!$this->user->isAdmin()) {
|
||||
throw new Forbidden();
|
||||
}
|
||||
|
||||
$timestamp = date('Ymd-His');
|
||||
$snapshotName = 'snapshot-' . $timestamp;
|
||||
$snapshotDir = 'data/backups/' . $snapshotName;
|
||||
|
||||
mkdir($snapshotDir, 0755, true);
|
||||
|
||||
$dbParams = $this->config->get('database');
|
||||
$host = $dbParams['host'] ?? 'localhost';
|
||||
$port = $dbParams['port'] ?? '3306';
|
||||
$dbName = $dbParams['dbname'] ?? '';
|
||||
$user = $dbParams['user'] ?? '';
|
||||
$password = $dbParams['password'] ?? '';
|
||||
|
||||
$dumpFile = $snapshotDir . '/database.sql';
|
||||
|
||||
$cmd = sprintf(
|
||||
'mysqldump --host=%s --port=%s --user=%s --password=%s'
|
||||
. ' --ssl=false --no-tablespaces --single-transaction --quick %s > %s 2>&1',
|
||||
escapeshellarg($host),
|
||||
escapeshellarg($port),
|
||||
escapeshellarg($user),
|
||||
escapeshellarg($password),
|
||||
escapeshellarg($dbName),
|
||||
escapeshellarg($dumpFile)
|
||||
);
|
||||
|
||||
$output = [];
|
||||
$returnCode = 0;
|
||||
exec($cmd, $output, $returnCode);
|
||||
|
||||
if ($returnCode !== 0) {
|
||||
// Cleanup on failure
|
||||
if (file_exists($dumpFile)) {
|
||||
unlink($dumpFile);
|
||||
}
|
||||
|
||||
rmdir($snapshotDir);
|
||||
|
||||
return ResponseComposer::json([
|
||||
'success' => false,
|
||||
'error' => 'Database dump failed: ' . implode("\n", $output),
|
||||
]);
|
||||
}
|
||||
|
||||
$dumpSize = filesize($dumpFile);
|
||||
|
||||
// Write a manifest
|
||||
$manifest = [
|
||||
'type' => 'snapshot',
|
||||
'createdAt' => date('c'),
|
||||
'sourceHost' => $this->config->get('siteUrl', 'unknown'),
|
||||
'espoVersion' => $this->config->get('version', 'unknown'),
|
||||
'database' => $dbName,
|
||||
'dumpSizeBytes' => $dumpSize,
|
||||
];
|
||||
|
||||
file_put_contents(
|
||||
$snapshotDir . '/manifest.json',
|
||||
json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
|
||||
);
|
||||
|
||||
return ResponseComposer::json([
|
||||
'success' => true,
|
||||
'snapshotName' => $snapshotName,
|
||||
'dumpSize' => $dumpSize,
|
||||
'dumpSizeFormatted' => $this->formatBytes($dumpSize),
|
||||
]);
|
||||
}
|
||||
|
||||
private function formatBytes(int $bytes): string
|
||||
{
|
||||
if ($bytes >= 1073741824) {
|
||||
return round($bytes / 1073741824, 2) . ' GB';
|
||||
}
|
||||
|
||||
if ($bytes >= 1048576) {
|
||||
return round($bytes / 1048576, 2) . ' MB';
|
||||
}
|
||||
|
||||
if ($bytes >= 1024) {
|
||||
return round($bytes / 1024, 2) . ' KB';
|
||||
}
|
||||
|
||||
return $bytes . ' B';
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,9 @@
|
||||
"Import failed": "Import failed",
|
||||
"Select a backup first": "Please select a backup first",
|
||||
"Failed to delete backup": "Failed to delete backup",
|
||||
"Failed to delete snapshot": "Failed to delete snapshot",
|
||||
"Database Snapshots": "Database Snapshots",
|
||||
"No snapshots": "No snapshots found",
|
||||
"Selected backup": "Selected backup",
|
||||
"Import Options": "Import Options",
|
||||
"Preserve original IDs": "Preserve original IDs",
|
||||
@@ -25,6 +28,10 @@
|
||||
"Exporting...": "Exporting...",
|
||||
"Importing...": "Importing...",
|
||||
"Log": "Log",
|
||||
"Create snapshot before import": "Create snapshot before import",
|
||||
"Creating snapshot...": "Creating database snapshot...",
|
||||
"Snapshot created": "Snapshot created",
|
||||
"Snapshot failed": "Snapshot failed",
|
||||
"records": "records",
|
||||
"files": "files",
|
||||
"optional": "optional",
|
||||
@@ -40,8 +47,11 @@
|
||||
"importConfirmation": "Are you sure you want to import this backup? This will add data to the database. This action cannot be easily undone.",
|
||||
"dryRunConfirmation": "Run a dry import? This will simulate the import without making any changes to the database.",
|
||||
"deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
|
||||
"deleteSnapshotConfirmation": "Are you sure you want to delete this database snapshot? This action cannot be undone.",
|
||||
"noFilesHint": "faster export, without attachment files",
|
||||
"preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
|
||||
"importNoFilesHint": "import only database records, skip attachment files",
|
||||
"snapshotHint": "backup the entire database before importing, so you can restore if needed",
|
||||
"skipDuplicatesHint": "skip records that already exist in the database"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,9 @@
|
||||
"Import failed": "הייבוא נכשל",
|
||||
"Select a backup first": "נא לבחור גיבוי קודם",
|
||||
"Failed to delete backup": "מחיקת הגיבוי נכשלה",
|
||||
"Failed to delete snapshot": "מחיקת צילום המצב נכשלה",
|
||||
"Database Snapshots": "צילומי מצב של מסד הנתונים",
|
||||
"No snapshots": "לא נמצאו צילומי מצב",
|
||||
"Selected backup": "גיבוי נבחר",
|
||||
"Import Options": "אפשרויות ייבוא",
|
||||
"Preserve original IDs": "שמור מזהים מקוריים",
|
||||
@@ -25,6 +28,10 @@
|
||||
"Exporting...": "מייצא...",
|
||||
"Importing...": "מייבא...",
|
||||
"Log": "לוג",
|
||||
"Create snapshot before import": "צור צילום מצב לפני ייבוא",
|
||||
"Creating snapshot...": "יוצר צילום מצב של מסד הנתונים...",
|
||||
"Snapshot created": "צילום מצב נוצר",
|
||||
"Snapshot failed": "יצירת צילום מצב נכשלה",
|
||||
"records": "רשומות",
|
||||
"files": "קבצים",
|
||||
"optional": "אופציונלי",
|
||||
@@ -40,8 +47,11 @@
|
||||
"importConfirmation": "האם אתה בטוח שברצונך לייבא גיבוי זה? פעולה זו תוסיף נתונים למסד הנתונים ולא ניתן לבטלה בקלות.",
|
||||
"dryRunConfirmation": "להריץ ייבוא ניסיון? פעולה זו תדמה את הייבוא מבלי לבצע שינויים במסד הנתונים.",
|
||||
"deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
|
||||
"deleteSnapshotConfirmation": "האם אתה בטוח שברצונך למחוק צילום מצב זה? לא ניתן לבטל פעולה זו.",
|
||||
"noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
|
||||
"preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
|
||||
"importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
|
||||
"snapshotHint": "גיבוי מלא של מסד הנתונים לפני הייבוא, כדי שתוכל לשחזר במקרה הצורך",
|
||||
"skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
{
|
||||
"customDev": {
|
||||
"label": "Custom Development",
|
||||
"customization": {
|
||||
"itemList": [
|
||||
"__APPEND__",
|
||||
{
|
||||
@@ -11,7 +10,6 @@
|
||||
"view": "modules/data-migration/views/admin/data-migration",
|
||||
"order": 50
|
||||
}
|
||||
],
|
||||
"order": 30
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,5 +4,6 @@
|
||||
"requires": {
|
||||
"espocrm": ">=8.0.0"
|
||||
},
|
||||
"description": "Data migration tool for case data backup and restore"
|
||||
"description": "Data migration tool for case data backup and restore",
|
||||
"clientModule": "data-migration"
|
||||
}
|
||||
|
||||
@@ -23,5 +23,10 @@
|
||||
"route": "/DataMigration/action/deleteBackup",
|
||||
"method": "post",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
|
||||
},
|
||||
{
|
||||
"route": "/DataMigration/action/snapshot",
|
||||
"method": "post",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Snapshot"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -309,11 +309,19 @@ class EntityImporter
|
||||
return $newId;
|
||||
}
|
||||
|
||||
/** @var array<string, string[]> Cache of existing column names per table */
|
||||
private array $tableColumnsCache = [];
|
||||
|
||||
/** @var array<string, array<string, array>> Cache of column metadata per table */
|
||||
private array $tableColumnMetaCache = [];
|
||||
|
||||
/** @var array<string, string[]> Skipped columns log per entity type */
|
||||
private array $skippedColumnsLog = [];
|
||||
|
||||
private function insertWithRawSql(string $entityType, array $record, string $newId): void
|
||||
{
|
||||
$tableName = $this->entityManager
|
||||
->getQueryComposer()
|
||||
->toDb($entityType);
|
||||
$tableName = $this->camelCaseToUnderscore($entityType);
|
||||
$existingColumns = $this->getTableColumns($tableName);
|
||||
|
||||
// Convert camelCase fields to snake_case for SQL
|
||||
$columns = [];
|
||||
@@ -327,15 +335,25 @@ class EntityImporter
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip non-storable array/object fields that come from getValueMap
|
||||
if (is_array($value) || is_object($value)) {
|
||||
// JSON fields should be stored as JSON strings
|
||||
if (is_array($value) || is_object($value)) {
|
||||
$value = json_encode($value, JSON_UNESCAPED_UNICODE);
|
||||
$column = $this->camelCaseToUnderscore($field);
|
||||
|
||||
// Skip columns that don't exist in the target table
|
||||
if (!in_array($column, $existingColumns)) {
|
||||
if (!isset($this->skippedColumnsLog[$entityType][$column])) {
|
||||
$this->skippedColumnsLog[$entityType][$column] = true;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$column = $this->camelCaseToUnderscore($field);
|
||||
// JSON fields should be stored as JSON strings
|
||||
if (is_array($value) || is_object($value)) {
|
||||
$value = json_encode($value, JSON_UNESCAPED_UNICODE);
|
||||
}
|
||||
|
||||
// Coerce value to match target column type
|
||||
$value = $this->coerceValue($tableName, $column, $value);
|
||||
|
||||
$columns[] = "`{$column}`";
|
||||
$values[] = $value;
|
||||
$placeholders[] = '?';
|
||||
@@ -346,14 +364,85 @@ class EntityImporter
|
||||
|
||||
$sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
|
||||
|
||||
try {
|
||||
$pdo = $this->entityManager->getPDO();
|
||||
$stmt = $pdo->prepare($sql);
|
||||
$stmt->execute($values);
|
||||
} catch (\Exception $e) {
|
||||
// If column mismatch, try with just known columns
|
||||
throw $e;
|
||||
$pdo = $this->entityManager->getPDO();
|
||||
$stmt = $pdo->prepare($sql);
|
||||
$stmt->execute($values);
|
||||
}
|
||||
|
||||
private function getTableColumns(string $tableName): array
|
||||
{
|
||||
if (isset($this->tableColumnsCache[$tableName])) {
|
||||
return $this->tableColumnsCache[$tableName];
|
||||
}
|
||||
|
||||
$pdo = $this->entityManager->getPDO();
|
||||
$sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
|
||||
$columns = [];
|
||||
$meta = [];
|
||||
|
||||
while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
|
||||
$columns[] = $row['Field'];
|
||||
$meta[$row['Field']] = [
|
||||
'type' => $row['Type'],
|
||||
'nullable' => $row['Null'] === 'YES',
|
||||
'default' => $row['Default'],
|
||||
];
|
||||
}
|
||||
|
||||
$this->tableColumnsCache[$tableName] = $columns;
|
||||
$this->tableColumnMetaCache[$tableName] = $meta;
|
||||
|
||||
return $columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce a value to be compatible with the target column type.
|
||||
*/
|
||||
private function coerceValue(string $tableName, string $column, mixed $value): mixed
|
||||
{
|
||||
$meta = $this->tableColumnMetaCache[$tableName][$column] ?? null;
|
||||
|
||||
if ($meta === null) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
$type = strtolower($meta['type']);
|
||||
$isNumeric = str_contains($type, 'int') || str_contains($type, 'decimal')
|
||||
|| str_contains($type, 'float') || str_contains($type, 'double')
|
||||
|| str_contains($type, 'tinyint') || str_contains($type, 'smallint')
|
||||
|| str_contains($type, 'bigint');
|
||||
|
||||
if ($isNumeric && ($value === '' || $value === false)) {
|
||||
if ($meta['nullable']) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!$meta['nullable'] && $value === null) {
|
||||
if ($isNumeric) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return $meta['default'] ?? '';
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string[]> entityType => [skipped column names]
|
||||
*/
|
||||
public function getSkippedColumns(): array
|
||||
{
|
||||
$result = [];
|
||||
|
||||
foreach ($this->skippedColumnsLog as $entityType => $columns) {
|
||||
$result[$entityType] = array_keys($columns);
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function resetAutoIncrement(string $entityType): void
|
||||
|
||||
@@ -62,6 +62,7 @@ class ImportService
|
||||
$preserveIds = $options['preserveIds'] ?? true;
|
||||
$skipDuplicates = $options['skipDuplicates'] ?? false;
|
||||
$dryRun = $options['dryRun'] ?? false;
|
||||
$noFiles = $options['noFiles'] ?? false;
|
||||
$verbose = $options['verbose'] ?? false;
|
||||
|
||||
$dataDir = rtrim($inputDir, '/') . '/data';
|
||||
@@ -138,7 +139,10 @@ class ImportService
|
||||
}
|
||||
|
||||
// Restore attachment files
|
||||
if (is_dir($filesDir)) {
|
||||
if ($noFiles) {
|
||||
$io->writeLine('');
|
||||
$io->writeLine('Attachment files: SKIPPED (--no-files)');
|
||||
} else if (is_dir($filesDir)) {
|
||||
$io->writeLine('');
|
||||
$io->writeLine('Restoring attachment files...');
|
||||
|
||||
@@ -175,6 +179,18 @@ class ImportService
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Report skipped columns (missing in target DB)
|
||||
$skippedColumns = $this->entityImporter->getSkippedColumns();
|
||||
|
||||
if (!empty($skippedColumns)) {
|
||||
$io->writeLine('');
|
||||
$io->writeLine('WARNING: Skipped columns not found in target database:');
|
||||
|
||||
foreach ($skippedColumns as $type => $columns) {
|
||||
$io->writeLine(" {$type}: " . implode(', ', $columns));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function entityTypeExists(string $entityType): bool
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "DataMigration",
|
||||
"version": "1.1.3",
|
||||
"version": "1.3.2",
|
||||
"acceptableVersions": [
|
||||
">=8.0.0"
|
||||
],
|
||||
"php": [
|
||||
">=8.1"
|
||||
],
|
||||
"releaseDate": "2026-03-29",
|
||||
"releaseDate": "2026-04-06",
|
||||
"author": "klear",
|
||||
"description": "Case data migration: export/import all case-related data between EspoCRM instances"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user