Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 258a5b8136 | |||
| 3dc5c4e571 | |||
| 8b745bc99f | |||
| 384d7c6df0 | |||
| 49275d0fc9 | |||
| ec759b89c1 | |||
| d5fe6c21a0 | |||
| 54e8c77325 | |||
| 35cda6e91c | |||
| ba00e7ecf6 | |||
| e3a1be1844 | |||
| fd279b6c8b | |||
| 73b5907539 | |||
| edeef544d4 | |||
| 1964ad99b6 | |||
| 078b2e6051 | |||
| a57e23dec3 | |||
| a8c4a2ebe1 | |||
| 412b1e643b | |||
| eed25a87b2 | |||
| 838ef5c2c6 | |||
| 319e2df45e |
@@ -116,6 +116,19 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{{! === Snapshots === }}
|
||||||
|
<div class="panel panel-info">
|
||||||
|
<div class="panel-heading">
|
||||||
|
<h4 class="panel-title">
|
||||||
|
<i class="fas fa-camera"></i>
|
||||||
|
{{translate 'Database Snapshots' scope='DataMigration'}}
|
||||||
|
</h4>
|
||||||
|
</div>
|
||||||
|
<div class="panel-body" style="padding: 0;">
|
||||||
|
<div class="snapshots-list list-group" style="margin-bottom: 0;"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{{! === Import === }}
|
{{! === Import === }}
|
||||||
<div class="import-section" style="display: none;">
|
<div class="import-section" style="display: none;">
|
||||||
<div class="panel panel-warning">
|
<div class="panel panel-warning">
|
||||||
@@ -150,6 +163,22 @@
|
|||||||
<span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
|
<span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="checkbox">
|
||||||
|
<label>
|
||||||
|
<input type="checkbox" name="importNoFiles">
|
||||||
|
{{translate 'Skip attachment files' scope='DataMigration'}}
|
||||||
|
<span class="text-muted"> — {{translate 'importNoFilesHint' category='messages' scope='DataMigration'}}</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="checkbox">
|
||||||
|
<label>
|
||||||
|
<input type="checkbox" name="createSnapshot" checked>
|
||||||
|
{{translate 'Create snapshot before import' scope='DataMigration'}}
|
||||||
|
<span class="text-muted"> — {{translate 'snapshotHint' category='messages' scope='DataMigration'}}</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="btn-group">
|
<div class="btn-group">
|
||||||
|
|||||||
@@ -72,6 +72,11 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
|||||||
const name = $(e.currentTarget).data('name');
|
const name = $(e.currentTarget).data('name');
|
||||||
this.$el.find('.backup-details[data-name="' + name + '"]').toggle();
|
this.$el.find('.backup-details[data-name="' + name + '"]').toggle();
|
||||||
},
|
},
|
||||||
|
'click [data-action="deleteSnapshot"]': function (e) {
|
||||||
|
e.stopPropagation();
|
||||||
|
const name = $(e.currentTarget).data('name');
|
||||||
|
this.actionDeleteSnapshot(name);
|
||||||
|
},
|
||||||
'click [data-action="dismissExportLog"]': function () {
|
'click [data-action="dismissExportLog"]': function () {
|
||||||
this.exportLog = null;
|
this.exportLog = null;
|
||||||
this.$el.find('.export-log-container').hide();
|
this.$el.find('.export-log-container').hide();
|
||||||
@@ -144,8 +149,10 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
|||||||
Espo.Ajax.getRequest('DataMigration/action/listBackups')
|
Espo.Ajax.getRequest('DataMigration/action/listBackups')
|
||||||
.then(response => {
|
.then(response => {
|
||||||
this.backups = response.backups || [];
|
this.backups = response.backups || [];
|
||||||
|
this.snapshots = response.snapshots || [];
|
||||||
this.isLoadingBackups = false;
|
this.isLoadingBackups = false;
|
||||||
this.renderBackups();
|
this.renderBackups();
|
||||||
|
this.renderSnapshots();
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
this.isLoadingBackups = false;
|
this.isLoadingBackups = false;
|
||||||
@@ -301,6 +308,8 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
|||||||
|
|
||||||
const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
|
const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
|
||||||
const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
|
const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
|
||||||
|
const noFiles = this.$el.find('input[name="importNoFiles"]').is(':checked');
|
||||||
|
const createSnapshot = !dryRun && this.$el.find('input[name="createSnapshot"]').is(':checked');
|
||||||
|
|
||||||
const confirmMsg = dryRun
|
const confirmMsg = dryRun
|
||||||
? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
|
? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
|
||||||
@@ -312,36 +321,72 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
|||||||
.addClass('disabled').attr('disabled', true);
|
.addClass('disabled').attr('disabled', true);
|
||||||
this.$el.find('.import-progress').show();
|
this.$el.find('.import-progress').show();
|
||||||
|
|
||||||
Espo.Ajax.postRequest('DataMigration/action/import', {
|
const runImport = () => {
|
||||||
backupName: this.selectedBackup,
|
Espo.Ajax.postRequest('DataMigration/action/import', {
|
||||||
preserveIds: preserveIds,
|
backupName: this.selectedBackup,
|
||||||
skipDuplicates: skipDuplicates,
|
preserveIds: preserveIds,
|
||||||
dryRun: dryRun,
|
noFiles: noFiles,
|
||||||
}, {timeout: 600000})
|
skipDuplicates: skipDuplicates,
|
||||||
.then(response => {
|
dryRun: dryRun,
|
||||||
this.isImporting = false;
|
}, {timeout: 600000})
|
||||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
.then(response => {
|
||||||
.removeClass('disabled').attr('disabled', false);
|
this.isImporting = false;
|
||||||
this.$el.find('.import-progress').hide();
|
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||||
|
.removeClass('disabled').attr('disabled', false);
|
||||||
|
this.$el.find('.import-progress').hide();
|
||||||
|
|
||||||
this.importLog = response.log || [];
|
this.importLog = response.log || [];
|
||||||
this.renderLog('.import-log-container', this.importLog, !dryRun);
|
this.renderLog('.import-log-container', this.importLog, !dryRun);
|
||||||
|
|
||||||
const label = dryRun ? 'Dry run completed' : 'Import completed';
|
const label = dryRun ? 'Dry run completed' : 'Import completed';
|
||||||
|
|
||||||
Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
|
Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
|
||||||
|
|
||||||
if (!dryRun) {
|
if (!dryRun) {
|
||||||
this.loadStatus();
|
this.loadStatus();
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.catch(xhr => {
|
.catch(xhr => {
|
||||||
this.isImporting = false;
|
this.isImporting = false;
|
||||||
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||||
.removeClass('disabled').attr('disabled', false);
|
.removeClass('disabled').attr('disabled', false);
|
||||||
this.$el.find('.import-progress').hide();
|
this.$el.find('.import-progress').hide();
|
||||||
Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
|
Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
|
||||||
});
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
if (createSnapshot) {
|
||||||
|
Espo.Ui.notify(this.translate('Creating snapshot...', 'labels', 'DataMigration'));
|
||||||
|
|
||||||
|
Espo.Ajax.postRequest('DataMigration/action/snapshot', {}, {timeout: 300000})
|
||||||
|
.then(response => {
|
||||||
|
if (response.success) {
|
||||||
|
Espo.Ui.success(
|
||||||
|
this.translate('Snapshot created', 'labels', 'DataMigration') +
|
||||||
|
': ' + response.snapshotName + ' (' + response.dumpSizeFormatted + ')'
|
||||||
|
);
|
||||||
|
runImport();
|
||||||
|
} else {
|
||||||
|
this.isImporting = false;
|
||||||
|
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||||
|
.removeClass('disabled').attr('disabled', false);
|
||||||
|
this.$el.find('.import-progress').hide();
|
||||||
|
Espo.Ui.error(
|
||||||
|
this.translate('Snapshot failed', 'labels', 'DataMigration') +
|
||||||
|
': ' + (response.error || 'Unknown error')
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(xhr => {
|
||||||
|
this.isImporting = false;
|
||||||
|
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
|
||||||
|
.removeClass('disabled').attr('disabled', false);
|
||||||
|
this.$el.find('.import-progress').hide();
|
||||||
|
Espo.Ui.error(this.translate('Snapshot failed', 'labels', 'DataMigration'));
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
runImport();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -368,6 +413,60 @@ define('modules/data-migration/views/admin/data-migration', ['view'], function (
|
|||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
renderSnapshots: function () {
|
||||||
|
const $container = this.$el.find('.snapshots-list');
|
||||||
|
|
||||||
|
if (!this.snapshots || this.snapshots.length === 0) {
|
||||||
|
$container.html(
|
||||||
|
'<div class="text-muted" style="padding: 15px; text-align: center;">' +
|
||||||
|
this.translate('No snapshots', 'labels', 'DataMigration') +
|
||||||
|
'</div>'
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let html = '';
|
||||||
|
|
||||||
|
this.snapshots.forEach(snap => {
|
||||||
|
const date = snap.createdAt ? new Date(snap.createdAt).toLocaleString() : 'Unknown';
|
||||||
|
|
||||||
|
html += '<div class="list-group-item">';
|
||||||
|
html += '<div class="pull-right">';
|
||||||
|
html += '<button class="btn btn-danger btn-xs" data-action="deleteSnapshot" data-name="' + snap.name + '" title="' + this.translate('Delete') + '">';
|
||||||
|
html += '<i class="fas fa-trash"></i></button>';
|
||||||
|
html += '</div>';
|
||||||
|
html += '<h5 class="list-group-item-heading" style="margin: 0 0 4px;">';
|
||||||
|
html += '<i class="fas fa-camera text-info"></i> ' + snap.name;
|
||||||
|
html += '</h5>';
|
||||||
|
html += '<p class="list-group-item-text text-muted" style="margin: 0; font-size: 0.9em;">';
|
||||||
|
html += '<i class="fas fa-clock"></i> ' + date;
|
||||||
|
html += ' | <i class="fas fa-database"></i> ' + snap.database;
|
||||||
|
html += ' | <i class="fas fa-hdd"></i> ' + snap.sizeFormatted;
|
||||||
|
html += '</p>';
|
||||||
|
html += '</div>';
|
||||||
|
});
|
||||||
|
|
||||||
|
$container.html(html);
|
||||||
|
},
|
||||||
|
|
||||||
|
actionDeleteSnapshot: function (name) {
|
||||||
|
this.confirm(
|
||||||
|
this.translate('deleteSnapshotConfirmation', 'messages', 'DataMigration'),
|
||||||
|
() => {
|
||||||
|
Espo.Ajax.postRequest('DataMigration/action/deleteBackup', {
|
||||||
|
backupName: name,
|
||||||
|
})
|
||||||
|
.then(() => {
|
||||||
|
Espo.Ui.success(this.translate('Deleted'));
|
||||||
|
this.loadBackups();
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
Espo.Ui.error(this.translate('Failed to delete snapshot', 'labels', 'DataMigration'));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
},
|
||||||
|
|
||||||
renderLog: function (selector, lines, isSuccess) {
|
renderLog: function (selector, lines, isSuccess) {
|
||||||
const $container = this.$el.find(selector);
|
const $container = this.$el.find(selector);
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ class DeleteBackup implements Action
|
|||||||
// Sanitize to prevent directory traversal
|
// Sanitize to prevent directory traversal
|
||||||
$safeName = basename($backupName);
|
$safeName = basename($backupName);
|
||||||
|
|
||||||
if (!str_starts_with($safeName, 'datamigration-')) {
|
if (!str_starts_with($safeName, 'datamigration-') && !str_starts_with($safeName, 'snapshot-')) {
|
||||||
throw new Forbidden('Invalid backup name.');
|
throw new Forbidden('Invalid backup name.');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ class Import implements Action
|
|||||||
$preserveIds = $body->preserveIds ?? true;
|
$preserveIds = $body->preserveIds ?? true;
|
||||||
$skipDuplicates = $body->skipDuplicates ?? true;
|
$skipDuplicates = $body->skipDuplicates ?? true;
|
||||||
$dryRun = $body->dryRun ?? false;
|
$dryRun = $body->dryRun ?? false;
|
||||||
|
$noFiles = $body->noFiles ?? false;
|
||||||
|
|
||||||
$io = new BufferedIO();
|
$io = new BufferedIO();
|
||||||
|
|
||||||
@@ -54,6 +55,7 @@ class Import implements Action
|
|||||||
'preserveIds' => $preserveIds,
|
'preserveIds' => $preserveIds,
|
||||||
'skipDuplicates' => $skipDuplicates,
|
'skipDuplicates' => $skipDuplicates,
|
||||||
'dryRun' => $dryRun,
|
'dryRun' => $dryRun,
|
||||||
|
'noFiles' => $noFiles,
|
||||||
'verbose' => true,
|
'verbose' => true,
|
||||||
], $io);
|
], $io);
|
||||||
|
|
||||||
|
|||||||
@@ -73,7 +73,43 @@ class ListBackups implements Action
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
return ResponseComposer::json(['backups' => $backups]);
|
// Scan for snapshots
|
||||||
|
$snapshots = [];
|
||||||
|
|
||||||
|
foreach ($dirs as $dir) {
|
||||||
|
if ($dir === '.' || $dir === '..') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!str_starts_with($dir, 'snapshot-')) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$fullPath = $backupsDir . '/' . $dir;
|
||||||
|
|
||||||
|
if (!is_dir($fullPath)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$manifest = $this->manifestBuilder->read($fullPath);
|
||||||
|
$dumpFile = $fullPath . '/database.sql';
|
||||||
|
$dumpSize = file_exists($dumpFile) ? filesize($dumpFile) : 0;
|
||||||
|
|
||||||
|
$snapshots[] = [
|
||||||
|
'name' => $dir,
|
||||||
|
'createdAt' => $manifest['createdAt'] ?? null,
|
||||||
|
'sourceHost' => $manifest['sourceHost'] ?? 'unknown',
|
||||||
|
'espoVersion' => $manifest['espoVersion'] ?? 'unknown',
|
||||||
|
'database' => $manifest['database'] ?? 'unknown',
|
||||||
|
'dumpSizeBytes' => $dumpSize,
|
||||||
|
'sizeFormatted' => $this->formatBytes($dumpSize),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
return ResponseComposer::json([
|
||||||
|
'backups' => $backups,
|
||||||
|
'snapshots' => $snapshots,
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function formatBytes(int $bytes): string
|
private function formatBytes(int $bytes): string
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Espo\Modules\DataMigration\Api;
|
||||||
|
|
||||||
|
use Espo\Core\Api\Action;
|
||||||
|
use Espo\Core\Api\Request;
|
||||||
|
use Espo\Core\Api\Response;
|
||||||
|
use Espo\Core\Api\ResponseComposer;
|
||||||
|
use Espo\Core\Exceptions\Forbidden;
|
||||||
|
use Espo\Core\Utils\Config;
|
||||||
|
use Espo\Entities\User;
|
||||||
|
|
||||||
|
class Snapshot implements Action
|
||||||
|
{
|
||||||
|
public function __construct(
|
||||||
|
private User $user,
|
||||||
|
private Config $config
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public function process(Request $request): Response
|
||||||
|
{
|
||||||
|
if (!$this->user->isAdmin()) {
|
||||||
|
throw new Forbidden();
|
||||||
|
}
|
||||||
|
|
||||||
|
$timestamp = date('Ymd-His');
|
||||||
|
$snapshotName = 'snapshot-' . $timestamp;
|
||||||
|
$snapshotDir = 'data/backups/' . $snapshotName;
|
||||||
|
|
||||||
|
mkdir($snapshotDir, 0755, true);
|
||||||
|
|
||||||
|
$dbParams = $this->config->get('database');
|
||||||
|
$host = $dbParams['host'] ?? 'localhost';
|
||||||
|
$port = $dbParams['port'] ?? '3306';
|
||||||
|
$dbName = $dbParams['dbname'] ?? '';
|
||||||
|
$user = $dbParams['user'] ?? '';
|
||||||
|
$password = $dbParams['password'] ?? '';
|
||||||
|
|
||||||
|
$dumpFile = $snapshotDir . '/database.sql';
|
||||||
|
|
||||||
|
$cmd = sprintf(
|
||||||
|
'mysqldump --host=%s --port=%s --user=%s --password=%s'
|
||||||
|
. ' --ssl=false --no-tablespaces --single-transaction --quick %s > %s 2>&1',
|
||||||
|
escapeshellarg($host),
|
||||||
|
escapeshellarg($port),
|
||||||
|
escapeshellarg($user),
|
||||||
|
escapeshellarg($password),
|
||||||
|
escapeshellarg($dbName),
|
||||||
|
escapeshellarg($dumpFile)
|
||||||
|
);
|
||||||
|
|
||||||
|
$output = [];
|
||||||
|
$returnCode = 0;
|
||||||
|
exec($cmd, $output, $returnCode);
|
||||||
|
|
||||||
|
if ($returnCode !== 0) {
|
||||||
|
// Cleanup on failure
|
||||||
|
if (file_exists($dumpFile)) {
|
||||||
|
unlink($dumpFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
rmdir($snapshotDir);
|
||||||
|
|
||||||
|
return ResponseComposer::json([
|
||||||
|
'success' => false,
|
||||||
|
'error' => 'Database dump failed: ' . implode("\n", $output),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
$dumpSize = filesize($dumpFile);
|
||||||
|
|
||||||
|
// Write a manifest
|
||||||
|
$manifest = [
|
||||||
|
'type' => 'snapshot',
|
||||||
|
'createdAt' => date('c'),
|
||||||
|
'sourceHost' => $this->config->get('siteUrl', 'unknown'),
|
||||||
|
'espoVersion' => $this->config->get('version', 'unknown'),
|
||||||
|
'database' => $dbName,
|
||||||
|
'dumpSizeBytes' => $dumpSize,
|
||||||
|
];
|
||||||
|
|
||||||
|
file_put_contents(
|
||||||
|
$snapshotDir . '/manifest.json',
|
||||||
|
json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
|
||||||
|
);
|
||||||
|
|
||||||
|
return ResponseComposer::json([
|
||||||
|
'success' => true,
|
||||||
|
'snapshotName' => $snapshotName,
|
||||||
|
'dumpSize' => $dumpSize,
|
||||||
|
'dumpSizeFormatted' => $this->formatBytes($dumpSize),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function formatBytes(int $bytes): string
|
||||||
|
{
|
||||||
|
if ($bytes >= 1073741824) {
|
||||||
|
return round($bytes / 1073741824, 2) . ' GB';
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($bytes >= 1048576) {
|
||||||
|
return round($bytes / 1048576, 2) . ' MB';
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($bytes >= 1024) {
|
||||||
|
return round($bytes / 1024, 2) . ' KB';
|
||||||
|
}
|
||||||
|
|
||||||
|
return $bytes . ' B';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,6 +16,9 @@
|
|||||||
"Import failed": "Import failed",
|
"Import failed": "Import failed",
|
||||||
"Select a backup first": "Please select a backup first",
|
"Select a backup first": "Please select a backup first",
|
||||||
"Failed to delete backup": "Failed to delete backup",
|
"Failed to delete backup": "Failed to delete backup",
|
||||||
|
"Failed to delete snapshot": "Failed to delete snapshot",
|
||||||
|
"Database Snapshots": "Database Snapshots",
|
||||||
|
"No snapshots": "No snapshots found",
|
||||||
"Selected backup": "Selected backup",
|
"Selected backup": "Selected backup",
|
||||||
"Import Options": "Import Options",
|
"Import Options": "Import Options",
|
||||||
"Preserve original IDs": "Preserve original IDs",
|
"Preserve original IDs": "Preserve original IDs",
|
||||||
@@ -25,6 +28,10 @@
|
|||||||
"Exporting...": "Exporting...",
|
"Exporting...": "Exporting...",
|
||||||
"Importing...": "Importing...",
|
"Importing...": "Importing...",
|
||||||
"Log": "Log",
|
"Log": "Log",
|
||||||
|
"Create snapshot before import": "Create snapshot before import",
|
||||||
|
"Creating snapshot...": "Creating database snapshot...",
|
||||||
|
"Snapshot created": "Snapshot created",
|
||||||
|
"Snapshot failed": "Snapshot failed",
|
||||||
"records": "records",
|
"records": "records",
|
||||||
"files": "files",
|
"files": "files",
|
||||||
"optional": "optional",
|
"optional": "optional",
|
||||||
@@ -40,8 +47,11 @@
|
|||||||
"importConfirmation": "Are you sure you want to import this backup? This will add data to the database. This action cannot be easily undone.",
|
"importConfirmation": "Are you sure you want to import this backup? This will add data to the database. This action cannot be easily undone.",
|
||||||
"dryRunConfirmation": "Run a dry import? This will simulate the import without making any changes to the database.",
|
"dryRunConfirmation": "Run a dry import? This will simulate the import without making any changes to the database.",
|
||||||
"deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
|
"deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
|
||||||
|
"deleteSnapshotConfirmation": "Are you sure you want to delete this database snapshot? This action cannot be undone.",
|
||||||
"noFilesHint": "faster export, without attachment files",
|
"noFilesHint": "faster export, without attachment files",
|
||||||
"preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
|
"preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
|
||||||
|
"importNoFilesHint": "import only database records, skip attachment files",
|
||||||
|
"snapshotHint": "backup the entire database before importing, so you can restore if needed",
|
||||||
"skipDuplicatesHint": "skip records that already exist in the database"
|
"skipDuplicatesHint": "skip records that already exist in the database"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
{
|
{
|
||||||
"labels": {
|
"labels": {
|
||||||
"Data Migration": "העברת נתונים"
|
"Data Migration": "העברת נתונים",
|
||||||
|
"Custom Development": "פיתוח עצמאי"
|
||||||
},
|
},
|
||||||
"descriptions": {
|
"descriptions": {
|
||||||
"dataMigration": "ייצוא וייבוא נתוני תיקים בין מערכות EspoCRM"
|
"dataMigration": "ייצוא וייבוא נתוני תיקים בין מערכות EspoCRM"
|
||||||
|
|||||||
@@ -16,6 +16,9 @@
|
|||||||
"Import failed": "הייבוא נכשל",
|
"Import failed": "הייבוא נכשל",
|
||||||
"Select a backup first": "נא לבחור גיבוי קודם",
|
"Select a backup first": "נא לבחור גיבוי קודם",
|
||||||
"Failed to delete backup": "מחיקת הגיבוי נכשלה",
|
"Failed to delete backup": "מחיקת הגיבוי נכשלה",
|
||||||
|
"Failed to delete snapshot": "מחיקת צילום המצב נכשלה",
|
||||||
|
"Database Snapshots": "צילומי מצב של מסד הנתונים",
|
||||||
|
"No snapshots": "לא נמצאו צילומי מצב",
|
||||||
"Selected backup": "גיבוי נבחר",
|
"Selected backup": "גיבוי נבחר",
|
||||||
"Import Options": "אפשרויות ייבוא",
|
"Import Options": "אפשרויות ייבוא",
|
||||||
"Preserve original IDs": "שמור מזהים מקוריים",
|
"Preserve original IDs": "שמור מזהים מקוריים",
|
||||||
@@ -25,6 +28,10 @@
|
|||||||
"Exporting...": "מייצא...",
|
"Exporting...": "מייצא...",
|
||||||
"Importing...": "מייבא...",
|
"Importing...": "מייבא...",
|
||||||
"Log": "לוג",
|
"Log": "לוג",
|
||||||
|
"Create snapshot before import": "צור צילום מצב לפני ייבוא",
|
||||||
|
"Creating snapshot...": "יוצר צילום מצב של מסד הנתונים...",
|
||||||
|
"Snapshot created": "צילום מצב נוצר",
|
||||||
|
"Snapshot failed": "יצירת צילום מצב נכשלה",
|
||||||
"records": "רשומות",
|
"records": "רשומות",
|
||||||
"files": "קבצים",
|
"files": "קבצים",
|
||||||
"optional": "אופציונלי",
|
"optional": "אופציונלי",
|
||||||
@@ -40,8 +47,11 @@
|
|||||||
"importConfirmation": "האם אתה בטוח שברצונך לייבא גיבוי זה? פעולה זו תוסיף נתונים למסד הנתונים ולא ניתן לבטלה בקלות.",
|
"importConfirmation": "האם אתה בטוח שברצונך לייבא גיבוי זה? פעולה זו תוסיף נתונים למסד הנתונים ולא ניתן לבטלה בקלות.",
|
||||||
"dryRunConfirmation": "להריץ ייבוא ניסיון? פעולה זו תדמה את הייבוא מבלי לבצע שינויים במסד הנתונים.",
|
"dryRunConfirmation": "להריץ ייבוא ניסיון? פעולה זו תדמה את הייבוא מבלי לבצע שינויים במסד הנתונים.",
|
||||||
"deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
|
"deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
|
||||||
|
"deleteSnapshotConfirmation": "האם אתה בטוח שברצונך למחוק צילום מצב זה? לא ניתן לבטל פעולה זו.",
|
||||||
"noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
|
"noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
|
||||||
"preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
|
"preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
|
||||||
|
"importNoFilesHint": "ייבוא רשומות בלבד, ללא קבצים מצורפים",
|
||||||
|
"snapshotHint": "גיבוי מלא של מסד הנתונים לפני הייבוא, כדי שתוכל לשחזר במקרה הצורך",
|
||||||
"skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
|
"skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"system": {
|
"customization": {
|
||||||
"itemList": [
|
"itemList": [
|
||||||
"__APPEND__",
|
"__APPEND__",
|
||||||
{
|
{
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
"iconClass": "fas fa-exchange-alt",
|
"iconClass": "fas fa-exchange-alt",
|
||||||
"description": "dataMigration",
|
"description": "dataMigration",
|
||||||
"view": "modules/data-migration/views/admin/data-migration",
|
"view": "modules/data-migration/views/admin/data-migration",
|
||||||
"order": 85
|
"order": 50
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,5 +4,6 @@
|
|||||||
"requires": {
|
"requires": {
|
||||||
"espocrm": ">=8.0.0"
|
"espocrm": ">=8.0.0"
|
||||||
},
|
},
|
||||||
"description": "Data migration tool for case data backup and restore"
|
"description": "Data migration tool for case data backup and restore",
|
||||||
|
"clientModule": "data-migration"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,5 +23,10 @@
|
|||||||
"route": "/DataMigration/action/deleteBackup",
|
"route": "/DataMigration/action/deleteBackup",
|
||||||
"method": "post",
|
"method": "post",
|
||||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
|
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"route": "/DataMigration/action/snapshot",
|
||||||
|
"method": "post",
|
||||||
|
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Snapshot"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -309,11 +309,19 @@ class EntityImporter
|
|||||||
return $newId;
|
return $newId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @var array<string, string[]> Cache of existing column names per table */
|
||||||
|
private array $tableColumnsCache = [];
|
||||||
|
|
||||||
|
/** @var array<string, array<string, array>> Cache of column metadata per table */
|
||||||
|
private array $tableColumnMetaCache = [];
|
||||||
|
|
||||||
|
/** @var array<string, string[]> Skipped columns log per entity type */
|
||||||
|
private array $skippedColumnsLog = [];
|
||||||
|
|
||||||
private function insertWithRawSql(string $entityType, array $record, string $newId): void
|
private function insertWithRawSql(string $entityType, array $record, string $newId): void
|
||||||
{
|
{
|
||||||
$tableName = $this->entityManager
|
$tableName = $this->camelCaseToUnderscore($entityType);
|
||||||
->getQueryComposer()
|
$existingColumns = $this->getTableColumns($tableName);
|
||||||
->toDb($entityType);
|
|
||||||
|
|
||||||
// Convert camelCase fields to snake_case for SQL
|
// Convert camelCase fields to snake_case for SQL
|
||||||
$columns = [];
|
$columns = [];
|
||||||
@@ -327,15 +335,25 @@ class EntityImporter
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip non-storable array/object fields that come from getValueMap
|
$column = $this->camelCaseToUnderscore($field);
|
||||||
if (is_array($value) || is_object($value)) {
|
|
||||||
// JSON fields should be stored as JSON strings
|
// Skip columns that don't exist in the target table
|
||||||
if (is_array($value) || is_object($value)) {
|
if (!in_array($column, $existingColumns)) {
|
||||||
$value = json_encode($value, JSON_UNESCAPED_UNICODE);
|
if (!isset($this->skippedColumnsLog[$entityType][$column])) {
|
||||||
|
$this->skippedColumnsLog[$entityType][$column] = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$column = $this->camelCaseToUnderscore($field);
|
// JSON fields should be stored as JSON strings
|
||||||
|
if (is_array($value) || is_object($value)) {
|
||||||
|
$value = json_encode($value, JSON_UNESCAPED_UNICODE);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coerce value to match target column type
|
||||||
|
$value = $this->coerceValue($tableName, $column, $value);
|
||||||
|
|
||||||
$columns[] = "`{$column}`";
|
$columns[] = "`{$column}`";
|
||||||
$values[] = $value;
|
$values[] = $value;
|
||||||
$placeholders[] = '?';
|
$placeholders[] = '?';
|
||||||
@@ -346,14 +364,85 @@ class EntityImporter
|
|||||||
|
|
||||||
$sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
|
$sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
|
||||||
|
|
||||||
try {
|
$pdo = $this->entityManager->getPDO();
|
||||||
$pdo = $this->entityManager->getPDO();
|
$stmt = $pdo->prepare($sql);
|
||||||
$stmt = $pdo->prepare($sql);
|
$stmt->execute($values);
|
||||||
$stmt->execute($values);
|
}
|
||||||
} catch (\Exception $e) {
|
|
||||||
// If column mismatch, try with just known columns
|
private function getTableColumns(string $tableName): array
|
||||||
throw $e;
|
{
|
||||||
|
if (isset($this->tableColumnsCache[$tableName])) {
|
||||||
|
return $this->tableColumnsCache[$tableName];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$pdo = $this->entityManager->getPDO();
|
||||||
|
$sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
|
||||||
|
$columns = [];
|
||||||
|
$meta = [];
|
||||||
|
|
||||||
|
while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
|
||||||
|
$columns[] = $row['Field'];
|
||||||
|
$meta[$row['Field']] = [
|
||||||
|
'type' => $row['Type'],
|
||||||
|
'nullable' => $row['Null'] === 'YES',
|
||||||
|
'default' => $row['Default'],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->tableColumnsCache[$tableName] = $columns;
|
||||||
|
$this->tableColumnMetaCache[$tableName] = $meta;
|
||||||
|
|
||||||
|
return $columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Coerce a value to be compatible with the target column type.
|
||||||
|
*/
|
||||||
|
private function coerceValue(string $tableName, string $column, mixed $value): mixed
|
||||||
|
{
|
||||||
|
$meta = $this->tableColumnMetaCache[$tableName][$column] ?? null;
|
||||||
|
|
||||||
|
if ($meta === null) {
|
||||||
|
return $value;
|
||||||
|
}
|
||||||
|
|
||||||
|
$type = strtolower($meta['type']);
|
||||||
|
$isNumeric = str_contains($type, 'int') || str_contains($type, 'decimal')
|
||||||
|
|| str_contains($type, 'float') || str_contains($type, 'double')
|
||||||
|
|| str_contains($type, 'tinyint') || str_contains($type, 'smallint')
|
||||||
|
|| str_contains($type, 'bigint');
|
||||||
|
|
||||||
|
if ($isNumeric && ($value === '' || $value === false)) {
|
||||||
|
if ($meta['nullable']) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$meta['nullable'] && $value === null) {
|
||||||
|
if ($isNumeric) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $meta['default'] ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
return $value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return array<string, string[]> entityType => [skipped column names]
|
||||||
|
*/
|
||||||
|
public function getSkippedColumns(): array
|
||||||
|
{
|
||||||
|
$result = [];
|
||||||
|
|
||||||
|
foreach ($this->skippedColumnsLog as $entityType => $columns) {
|
||||||
|
$result[$entityType] = array_keys($columns);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function resetAutoIncrement(string $entityType): void
|
private function resetAutoIncrement(string $entityType): void
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ class ImportService
|
|||||||
$preserveIds = $options['preserveIds'] ?? true;
|
$preserveIds = $options['preserveIds'] ?? true;
|
||||||
$skipDuplicates = $options['skipDuplicates'] ?? false;
|
$skipDuplicates = $options['skipDuplicates'] ?? false;
|
||||||
$dryRun = $options['dryRun'] ?? false;
|
$dryRun = $options['dryRun'] ?? false;
|
||||||
|
$noFiles = $options['noFiles'] ?? false;
|
||||||
$verbose = $options['verbose'] ?? false;
|
$verbose = $options['verbose'] ?? false;
|
||||||
|
|
||||||
$dataDir = rtrim($inputDir, '/') . '/data';
|
$dataDir = rtrim($inputDir, '/') . '/data';
|
||||||
@@ -100,6 +101,11 @@ class ImportService
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!$this->entityTypeExists($entityType)) {
|
||||||
|
$io->writeLine("Skipping {$entityType} (module not installed)");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
$io->writeLine("Importing {$entityType}...");
|
$io->writeLine("Importing {$entityType}...");
|
||||||
|
|
||||||
$result = $this->entityImporter->import(
|
$result = $this->entityImporter->import(
|
||||||
@@ -133,7 +139,10 @@ class ImportService
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Restore attachment files
|
// Restore attachment files
|
||||||
if (is_dir($filesDir)) {
|
if ($noFiles) {
|
||||||
|
$io->writeLine('');
|
||||||
|
$io->writeLine('Attachment files: SKIPPED (--no-files)');
|
||||||
|
} else if (is_dir($filesDir)) {
|
||||||
$io->writeLine('');
|
$io->writeLine('');
|
||||||
$io->writeLine('Restoring attachment files...');
|
$io->writeLine('Restoring attachment files...');
|
||||||
|
|
||||||
@@ -170,5 +179,27 @@ class ImportService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Report skipped columns (missing in target DB)
|
||||||
|
$skippedColumns = $this->entityImporter->getSkippedColumns();
|
||||||
|
|
||||||
|
if (!empty($skippedColumns)) {
|
||||||
|
$io->writeLine('');
|
||||||
|
$io->writeLine('WARNING: Skipped columns not found in target database:');
|
||||||
|
|
||||||
|
foreach ($skippedColumns as $type => $columns) {
|
||||||
|
$io->writeLine(" {$type}: " . implode(', ', $columns));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function entityTypeExists(string $entityType): bool
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
$this->entityManager->getRDBRepository($entityType);
|
||||||
|
return true;
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -1,13 +1,13 @@
|
|||||||
{
|
{
|
||||||
"name": "DataMigration",
|
"name": "DataMigration",
|
||||||
"version": "1.1.1",
|
"version": "1.3.2",
|
||||||
"acceptableVersions": [
|
"acceptableVersions": [
|
||||||
">=8.0.0"
|
">=8.0.0"
|
||||||
],
|
],
|
||||||
"php": [
|
"php": [
|
||||||
">=8.1"
|
">=8.1"
|
||||||
],
|
],
|
||||||
"releaseDate": "2026-03-29",
|
"releaseDate": "2026-04-06",
|
||||||
"author": "klear",
|
"author": "klear",
|
||||||
"description": "Case data migration: export/import all case-related data between EspoCRM instances"
|
"description": "Case data migration: export/import all case-related data between EspoCRM instances"
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user