feat: add admin UI for export/import data migration

- Admin panel page at #Admin/dataMigration with two-column layout
- Export: filter by case status, skip attachments option, progress + log
- Import: select backup, preserve IDs / skip duplicates, dry run support
- List/delete existing backups with full manifest details
- 5 API endpoints (status, export, import, listBackups, deleteBackup)
- BufferedIO adapter to capture CLI output for API responses
- Admin-only access on all endpoints
- i18n: English + Hebrew (fa_IR) translations

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-03-29 19:45:20 +00:00
parent e08eca7eb3
commit 6d6f5d571a
15 changed files with 1211 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
*.zip
@@ -0,0 +1,179 @@
<div class="page-header">
<h3>
<a href="#Admin">{{translate 'Administration'}}</a>
<span class="breadcrumb-separator"><span class="chevron-right"></span></span>
{{translate 'Data Migration' scope='DataMigration'}}
</h3>
</div>
<div class="row">
{{! ========== LEFT COLUMN: Status + Export ========== }}
<div class="col-md-6">
{{! === Database Status === }}
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
<button class="btn btn-default btn-xs" data-action="refreshStatus" title="{{translate 'Refresh'}}">
<i class="fas fa-sync-alt"></i>
</button>
</div>
<h4 class="panel-title">
<i class="fas fa-database"></i>
{{translate 'Database Status' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body">
<div class="status-spinner" style="text-align: center; padding: 20px;">
<i class="fas fa-spinner fa-spin fa-2x text-muted"></i>
</div>
<div class="status-content" style="display: none;"></div>
</div>
</div>
{{! === Export === }}
<div class="panel panel-primary">
<div class="panel-heading">
<h4 class="panel-title">
<i class="fas fa-upload"></i>
{{translate 'Export / Backup' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body">
<p class="text-muted" style="margin-bottom: 15px;">
{{translate 'exportDescription' category='messages' scope='DataMigration'}}
</p>
{{! Status filter }}
<div class="form-group">
<label class="control-label">
{{translate 'Filter by Case Status' scope='DataMigration'}}
<span class="text-muted" style="font-weight: normal;"> ({{translate 'optional' scope='DataMigration'}})</span>
</label>
<div class="export-status-checkboxes" style="columns: 2;">
{{#each caseStatuses}}
<div class="checkbox" style="margin-top: 2px; margin-bottom: 2px;">
<label>
<input type="checkbox" value="{{this}}"> {{this}}
</label>
</div>
{{/each}}
</div>
</div>
{{! No files option }}
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" name="noFiles">
{{translate 'Skip attachment files' scope='DataMigration'}}
<span class="text-muted"> — {{translate 'noFilesHint' category='messages' scope='DataMigration'}}</span>
</label>
</div>
</div>
<div>
<button class="btn btn-primary" data-action="export">
<i class="fas fa-download"></i>
{{translate 'Start Export' scope='DataMigration'}}
</button>
</div>
<div class="export-progress" style="display: none; margin-top: 15px;">
<div class="progress">
<div class="progress-bar progress-bar-striped active" style="width: 100%;">
{{translate 'Exporting...' scope='DataMigration'}}
</div>
</div>
</div>
<div class="export-log-container" style="display: none;"></div>
</div>
</div>
</div>
{{! ========== RIGHT COLUMN: Backups + Import ========== }}
<div class="col-md-6">
{{! === Backup List === }}
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
<button class="btn btn-default btn-xs" data-action="refreshBackups" title="{{translate 'Refresh'}}">
<i class="fas fa-sync-alt"></i>
</button>
</div>
<h4 class="panel-title">
<i class="fas fa-archive"></i>
{{translate 'Available Backups' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body" style="padding: 0;">
<div class="backups-spinner" style="text-align: center; padding: 20px;">
<i class="fas fa-spinner fa-spin fa-2x text-muted"></i>
</div>
<div class="backups-list list-group" style="margin-bottom: 0;"></div>
</div>
</div>
{{! === Import === }}
<div class="import-section" style="display: none;">
<div class="panel panel-warning">
<div class="panel-heading">
<h4 class="panel-title">
<i class="fas fa-download"></i>
{{translate 'Import / Restore' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body">
<div class="alert alert-info" style="margin-bottom: 15px;">
<i class="fas fa-info-circle"></i>
{{translate 'Selected backup' scope='DataMigration'}}: <strong class="selected-backup-name"></strong>
</div>
{{! Import options }}
<div class="form-group">
<label class="control-label">{{translate 'Import Options' scope='DataMigration'}}</label>
<div class="checkbox">
<label>
<input type="checkbox" name="preserveIds" checked>
{{translate 'Preserve original IDs' scope='DataMigration'}}
<span class="text-muted"> — {{translate 'preserveIdsHint' category='messages' scope='DataMigration'}}</span>
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="skipDuplicates" checked>
{{translate 'Skip duplicates' scope='DataMigration'}}
<span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
</label>
</div>
</div>
<div class="btn-group">
<button class="btn btn-warning" data-action="dryRunImport">
<i class="fas fa-vial"></i>
{{translate 'Dry Run' scope='DataMigration'}}
</button>
<button class="btn btn-danger" data-action="import">
<i class="fas fa-play"></i>
{{translate 'Import Now' scope='DataMigration'}}
</button>
</div>
<div class="import-progress" style="display: none; margin-top: 15px;">
<div class="progress">
<div class="progress-bar progress-bar-warning progress-bar-striped active" style="width: 100%;">
{{translate 'Importing...' scope='DataMigration'}}
</div>
</div>
</div>
<div class="import-log-container" style="display: none;"></div>
</div>
</div>
</div>
</div>
</div>
@@ -0,0 +1,427 @@
define('modules/data-migration/views/admin/data-migration', ['view'], function (Dep) {
const Parent = Dep.__esModule ? Dep.default : Dep;
return Parent.extend({
template: 'data-migration:admin/data-migration',
isExporting: false,
isImporting: false,
isLoadingStatus: false,
isLoadingBackups: false,
data: function () {
return {
isExporting: this.isExporting,
isImporting: this.isImporting,
isLoadingStatus: this.isLoadingStatus,
statusData: this.statusData,
backups: this.backups,
selectedBackup: this.selectedBackup,
exportLog: this.exportLog,
importLog: this.importLog,
caseStatuses: this.caseStatuses,
};
},
setup: function () {
this.statusData = null;
this.backups = [];
this.selectedBackup = null;
this.exportLog = null;
this.importLog = null;
this.caseStatuses = [
'Active', 'Pending', 'Closed', 'Rejected',
'On Hold', 'Won', 'Lost', 'Archived',
];
this.once('after:render', () => {
this.loadStatus();
this.loadBackups();
});
},
events: {
'click [data-action="export"]': function () {
this.actionExport();
},
'click [data-action="import"]': function () {
this.actionImport();
},
'click [data-action="dryRunImport"]': function () {
this.actionImport(true);
},
'click [data-action="refreshStatus"]': function () {
this.loadStatus();
},
'click [data-action="refreshBackups"]': function () {
this.loadBackups();
},
'click [data-action="selectBackup"]': function (e) {
const name = $(e.currentTarget).data('name');
this.selectBackup(name);
},
'click [data-action="deleteBackup"]': function (e) {
e.stopPropagation();
const name = $(e.currentTarget).data('name');
this.actionDeleteBackup(name);
},
'click [data-action="toggleBackupDetails"]': function (e) {
const name = $(e.currentTarget).data('name');
this.$el.find('.backup-details[data-name="' + name + '"]').toggle();
},
'click [data-action="dismissExportLog"]': function () {
this.exportLog = null;
this.$el.find('.export-log-container').hide();
},
'click [data-action="dismissImportLog"]': function () {
this.importLog = null;
this.$el.find('.import-log-container').hide();
},
},
loadStatus: function () {
this.isLoadingStatus = true;
this.$el.find('.status-spinner').show();
this.$el.find('.status-content').hide();
Espo.Ajax.getRequest('DataMigration/action/status')
.then(response => {
this.statusData = response;
this.isLoadingStatus = false;
this.renderStatus();
})
.catch(() => {
this.isLoadingStatus = false;
this.$el.find('.status-spinner').hide();
Espo.Ui.error('Failed to load status');
});
},
renderStatus: function () {
const $container = this.$el.find('.status-content');
const $spinner = this.$el.find('.status-spinner');
$spinner.hide();
if (!this.statusData) {
$container.html('<p class="text-muted">No data</p>').show();
return;
}
let html = '<table class="table table-bordered table-sm">';
html += '<thead><tr><th>' + this.translate('Entity Type') + '</th><th class="text-right">' + this.translate('Count') + '</th></tr></thead>';
html += '<tbody>';
let total = 0;
const counts = this.statusData.entityCounts || {};
for (const type in counts) {
if (counts[type] > 0) {
html += '<tr><td>' + type + '</td><td class="text-right">' + counts[type].toLocaleString() + '</td></tr>';
total += counts[type];
}
}
html += '<tr class="active"><td><strong>' + this.translate('Total') + '</strong></td>';
html += '<td class="text-right"><strong>' + total.toLocaleString() + '</strong></td></tr>';
html += '</tbody></table>';
html += '<p class="text-muted" style="margin-top: 8px;">';
html += '<i class="fas fa-paperclip"></i> ';
html += this.translate('Attachments') + ': ' + (this.statusData.attachmentFiles || 0).toLocaleString();
html += ' (' + (this.statusData.attachmentSizeFormatted || '0 B') + ')';
html += '</p>';
$container.html(html).show();
},
loadBackups: function () {
this.isLoadingBackups = true;
this.$el.find('.backups-spinner').show();
Espo.Ajax.getRequest('DataMigration/action/listBackups')
.then(response => {
this.backups = response.backups || [];
this.isLoadingBackups = false;
this.renderBackups();
})
.catch(() => {
this.isLoadingBackups = false;
this.$el.find('.backups-spinner').hide();
Espo.Ui.error('Failed to load backups');
});
},
renderBackups: function () {
const $container = this.$el.find('.backups-list');
const $spinner = this.$el.find('.backups-spinner');
$spinner.hide();
if (!this.backups || this.backups.length === 0) {
$container.html(
'<div class="no-data text-muted" style="padding: 20px; text-align: center;">' +
'<i class="fas fa-archive" style="font-size: 32px; margin-bottom: 10px; display: block;"></i>' +
this.translate('No backups found', 'labels', 'DataMigration') +
'</div>'
);
return;
}
let html = '';
this.backups.forEach(backup => {
const isSelected = this.selectedBackup === backup.name;
const selectedClass = isSelected ? ' active' : '';
const date = backup.exportedAt ? new Date(backup.exportedAt).toLocaleString() : 'Unknown';
let totalRecords = 0;
const ec = backup.entityCounts || {};
for (const k in ec) {
totalRecords += ec[k];
}
html += '<div class="backup-item list-group-item' + selectedClass + '" data-action="selectBackup" data-name="' + backup.name + '" style="cursor: pointer;">';
html += '<div class="pull-right">';
html += '<button class="btn btn-danger btn-xs" data-action="deleteBackup" data-name="' + backup.name + '" title="' + this.translate('Delete') + '">';
html += '<i class="fas fa-trash"></i></button>';
html += '</div>';
html += '<h5 class="list-group-item-heading" style="margin: 0 0 4px;">';
html += '<i class="fas fa-archive text-muted"></i> ';
html += '<span data-action="toggleBackupDetails" data-name="' + backup.name + '">' + backup.name + '</span>';
html += '</h5>';
html += '<p class="list-group-item-text text-muted" style="margin: 0; font-size: 0.9em;">';
html += '<i class="fas fa-clock"></i> ' + date;
html += ' &nbsp;|&nbsp; <i class="fas fa-database"></i> ' + totalRecords.toLocaleString() + ' ' + this.translate('records', 'labels', 'DataMigration');
html += ' &nbsp;|&nbsp; <i class="fas fa-paperclip"></i> ' + (backup.totalAttachmentFiles || 0).toLocaleString() + ' ' + this.translate('files', 'labels', 'DataMigration');
html += ' (' + (backup.sizeFormatted || '0 B') + ')';
html += '</p>';
// Details (hidden by default)
html += '<div class="backup-details" data-name="' + backup.name + '" style="display: none; margin-top: 10px;">';
html += '<table class="table table-condensed table-sm" style="margin-bottom: 0;">';
for (const et in ec) {
if (ec[et] > 0) {
html += '<tr><td style="padding: 2px 8px;">' + et + '</td><td style="padding: 2px 8px;" class="text-right">' + ec[et].toLocaleString() + '</td></tr>';
}
}
html += '</table>';
if (backup.modules && Object.keys(backup.modules).length > 0) {
html += '<div class="text-muted" style="margin-top: 5px; font-size: 0.85em;">';
html += '<strong>' + this.translate('Modules') + ':</strong> ';
const moduleParts = [];
for (const mod in backup.modules) {
moduleParts.push(mod + ' v' + backup.modules[mod]);
}
html += moduleParts.join(', ');
html += '</div>';
}
html += '</div>';
html += '</div>';
});
$container.html(html);
},
selectBackup: function (name) {
if (this.selectedBackup === name) {
this.selectedBackup = null;
} else {
this.selectedBackup = name;
}
this.$el.find('.backup-item').removeClass('active');
if (this.selectedBackup) {
this.$el.find('.backup-item[data-name="' + name + '"]').addClass('active');
this.$el.find('.import-section').show();
this.$el.find('.selected-backup-name').text(name);
} else {
this.$el.find('.import-section').hide();
}
},
actionExport: function () {
const statusFilters = [];
this.$el.find('.export-status-checkboxes input:checked').each(function () {
statusFilters.push($(this).val());
});
const noFiles = this.$el.find('input[name="noFiles"]').is(':checked');
this.confirm(
this.translate('exportConfirmation', 'messages', 'DataMigration'),
() => {
this.isExporting = true;
this.$el.find('[data-action="export"]').addClass('disabled').attr('disabled', true);
this.$el.find('.export-progress').show();
Espo.Ajax.postRequest('DataMigration/action/export', {
caseStatus: statusFilters.length > 0 ? statusFilters : null,
noFiles: noFiles,
}, {timeout: 600000})
.then(response => {
this.isExporting = false;
this.$el.find('[data-action="export"]').removeClass('disabled').attr('disabled', false);
this.$el.find('.export-progress').hide();
this.exportLog = response.log || [];
this.renderLog('.export-log-container', this.exportLog, true);
Espo.Ui.success(this.translate('Export completed', 'labels', 'DataMigration'));
this.loadBackups();
this.loadStatus();
})
.catch(xhr => {
this.isExporting = false;
this.$el.find('[data-action="export"]').removeClass('disabled').attr('disabled', false);
this.$el.find('.export-progress').hide();
Espo.Ui.error(this.translate('Export failed', 'labels', 'DataMigration'));
});
}
);
},
actionImport: function (dryRun) {
dryRun = dryRun || false;
if (!this.selectedBackup) {
Espo.Ui.warning(this.translate('Select a backup first', 'labels', 'DataMigration'));
return;
}
const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
const confirmMsg = dryRun
? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
: this.translate('importConfirmation', 'messages', 'DataMigration');
this.confirm(confirmMsg, () => {
this.isImporting = true;
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
.addClass('disabled').attr('disabled', true);
this.$el.find('.import-progress').show();
Espo.Ajax.postRequest('DataMigration/action/import', {
backupName: this.selectedBackup,
preserveIds: preserveIds,
skipDuplicates: skipDuplicates,
dryRun: dryRun,
}, {timeout: 600000})
.then(response => {
this.isImporting = false;
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
.removeClass('disabled').attr('disabled', false);
this.$el.find('.import-progress').hide();
this.importLog = response.log || [];
this.renderLog('.import-log-container', this.importLog, !dryRun);
const label = dryRun ? 'Dry run completed' : 'Import completed';
Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
if (!dryRun) {
this.loadStatus();
}
})
.catch(xhr => {
this.isImporting = false;
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
.removeClass('disabled').attr('disabled', false);
this.$el.find('.import-progress').hide();
Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
});
});
},
actionDeleteBackup: function (name) {
this.confirm(
this.translate('deleteBackupConfirmation', 'messages', 'DataMigration'),
() => {
Espo.Ajax.postRequest('DataMigration/action/deleteBackup', {
backupName: name,
})
.then(() => {
if (this.selectedBackup === name) {
this.selectedBackup = null;
this.$el.find('.import-section').hide();
}
Espo.Ui.success(this.translate('Deleted'));
this.loadBackups();
})
.catch(() => {
Espo.Ui.error(this.translate('Failed to delete backup', 'labels', 'DataMigration'));
});
}
);
},
renderLog: function (selector, lines, isSuccess) {
const $container = this.$el.find(selector);
if (!lines || lines.length === 0) {
$container.hide();
return;
}
const panelClass = isSuccess ? 'panel-success' : 'panel-info';
const dismissAction = selector.includes('export') ? 'dismissExportLog' : 'dismissImportLog';
let html = '<div class="panel ' + panelClass + '" style="margin-top: 15px;">';
html += '<div class="panel-heading">';
html += '<button type="button" class="close" data-action="' + dismissAction + '">&times;</button>';
html += '<h4 class="panel-title"><i class="fas fa-terminal"></i> ' + this.translate('Log', 'labels', 'DataMigration') + '</h4>';
html += '</div>';
html += '<div class="panel-body" style="max-height: 400px; overflow-y: auto;">';
html += '<pre style="margin: 0; white-space: pre-wrap; font-size: 12px; background: transparent; border: none; padding: 0;">';
lines.forEach(line => {
let lineClass = '';
if (line.includes('ERROR')) {
lineClass = 'text-danger';
} else if (line.includes('WARNING')) {
lineClass = 'text-warning';
} else if (line.includes('===') || line.includes('Complete')) {
lineClass = 'text-success';
}
if (lineClass) {
html += '<span class="' + lineClass + '">' + this.escapeHtml(line) + '</span>\n';
} else {
html += this.escapeHtml(line) + '\n';
}
});
html += '</pre></div></div>';
$container.html(html).show();
},
escapeHtml: function (text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
},
confirm: function (message, callback) {
Espo.Ui.confirm(message, {
confirmText: this.translate('Yes'),
cancelText: this.translate('No'),
}, callback);
},
});
});
@@ -0,0 +1,73 @@
<?php
namespace Espo\Modules\DataMigration\Api;
use Espo\Core\Api\Action;
use Espo\Core\Api\Request;
use Espo\Core\Api\Response;
use Espo\Core\Api\ResponseComposer;
use Espo\Core\Exceptions\BadRequest;
use Espo\Core\Exceptions\Forbidden;
use Espo\Entities\User;
class DeleteBackup implements Action
{
public function __construct(
private User $user
) {}
public function process(Request $request): Response
{
if (!$this->user->isAdmin()) {
throw new Forbidden();
}
$body = $request->getParsedBody();
$backupName = $body->backupName ?? null;
if (!$backupName) {
throw new BadRequest('backupName is required.');
}
// Sanitize to prevent directory traversal
$safeName = basename($backupName);
if (!str_starts_with($safeName, 'datamigration-')) {
throw new Forbidden('Invalid backup name.');
}
$fullPath = 'data/backups/' . $safeName;
if (!is_dir($fullPath)) {
throw new BadRequest("Backup not found: {$safeName}");
}
$this->deleteDirectory($fullPath);
return ResponseComposer::json([
'success' => true,
'deleted' => $safeName,
]);
}
private function deleteDirectory(string $dir): void
{
$items = scandir($dir);
foreach ($items as $item) {
if ($item === '.' || $item === '..') {
continue;
}
$path = $dir . '/' . $item;
if (is_dir($path)) {
$this->deleteDirectory($path);
} else {
unlink($path);
}
}
rmdir($dir);
}
}
@@ -0,0 +1,53 @@
<?php
namespace Espo\Modules\DataMigration\Api;
use Espo\Core\Api\Action;
use Espo\Core\Api\Request;
use Espo\Core\Api\Response;
use Espo\Core\Api\ResponseComposer;
use Espo\Core\Exceptions\Forbidden;
use Espo\Entities\User;
use Espo\Modules\DataMigration\Services\ExportService;
use Espo\Modules\DataMigration\Services\BufferedIO;
class Export implements Action
{
public function __construct(
private ExportService $exportService,
private User $user
) {}
public function process(Request $request): Response
{
if (!$this->user->isAdmin()) {
throw new Forbidden();
}
$body = $request->getParsedBody();
$caseIds = $body->caseIds ?? null;
$caseStatus = $body->caseStatus ?? null;
$noFiles = $body->noFiles ?? false;
$timestamp = date('Ymd-His');
$outputDir = 'data/backups/datamigration-' . $timestamp;
$io = new BufferedIO();
$this->exportService->export($outputDir, [
'batchSize' => 500,
'caseIds' => $caseIds,
'caseStatus' => is_array($caseStatus) && !empty($caseStatus) ? $caseStatus : null,
'noFiles' => $noFiles,
'verbose' => true,
], $io);
return ResponseComposer::json([
'success' => true,
'outputDir' => $outputDir,
'backupName' => 'datamigration-' . $timestamp,
'log' => $io->getLines(),
]);
}
}
@@ -0,0 +1,67 @@
<?php
namespace Espo\Modules\DataMigration\Api;
use Espo\Core\Api\Action;
use Espo\Core\Api\Request;
use Espo\Core\Api\Response;
use Espo\Core\Api\ResponseComposer;
use Espo\Core\Exceptions\BadRequest;
use Espo\Core\Exceptions\Forbidden;
use Espo\Entities\User;
use Espo\Modules\DataMigration\Services\ImportService;
use Espo\Modules\DataMigration\Services\BufferedIO;
class Import implements Action
{
public function __construct(
private ImportService $importService,
private User $user
) {}
public function process(Request $request): Response
{
if (!$this->user->isAdmin()) {
throw new Forbidden();
}
$body = $request->getParsedBody();
$backupName = $body->backupName ?? null;
if (!$backupName) {
throw new BadRequest('backupName is required.');
}
$inputDir = 'data/backups/' . basename($backupName);
if (!is_dir($inputDir)) {
throw new BadRequest("Backup not found: {$backupName}");
}
if (!file_exists($inputDir . '/manifest.json')) {
throw new BadRequest("Invalid backup: no manifest.json found.");
}
$preserveIds = $body->preserveIds ?? true;
$skipDuplicates = $body->skipDuplicates ?? true;
$dryRun = $body->dryRun ?? false;
$io = new BufferedIO();
$this->importService->import($inputDir, [
'batchSize' => 500,
'preserveIds' => $preserveIds,
'skipDuplicates' => $skipDuplicates,
'dryRun' => $dryRun,
'verbose' => true,
], $io);
return ResponseComposer::json([
'success' => true,
'backupName' => $backupName,
'dryRun' => $dryRun,
'log' => $io->getLines(),
]);
}
}
@@ -0,0 +1,95 @@
<?php
namespace Espo\Modules\DataMigration\Api;
use Espo\Core\Api\Action;
use Espo\Core\Api\Request;
use Espo\Core\Api\Response;
use Espo\Core\Api\ResponseComposer;
use Espo\Core\Exceptions\Forbidden;
use Espo\Entities\User;
use Espo\Modules\DataMigration\Services\ManifestBuilder;
class ListBackups implements Action
{
public function __construct(
private ManifestBuilder $manifestBuilder,
private User $user
) {}
public function process(Request $request): Response
{
if (!$this->user->isAdmin()) {
throw new Forbidden();
}
$backupsDir = 'data/backups';
$backups = [];
if (!is_dir($backupsDir)) {
return ResponseComposer::json(['backups' => []]);
}
$dirs = scandir($backupsDir, SCANDIR_SORT_DESCENDING);
foreach ($dirs as $dir) {
if ($dir === '.' || $dir === '..') {
continue;
}
if (!str_starts_with($dir, 'datamigration-')) {
continue;
}
$fullPath = $backupsDir . '/' . $dir;
if (!is_dir($fullPath)) {
continue;
}
$manifest = $this->manifestBuilder->read($fullPath);
if ($manifest === null) {
continue;
}
$totalRecords = 0;
foreach ($manifest['entityCounts'] ?? [] as $count) {
$totalRecords += $count;
}
$backups[] = [
'name' => $dir,
'exportedAt' => $manifest['exportedAt'] ?? null,
'sourceHost' => $manifest['sourceHost'] ?? 'unknown',
'espoVersion' => $manifest['espoVersion'] ?? 'unknown',
'totalRecords' => $totalRecords,
'totalAttachmentFiles' => $manifest['totalAttachmentFiles'] ?? 0,
'totalAttachmentSizeBytes' => $manifest['totalAttachmentSizeBytes'] ?? 0,
'entityCounts' => $manifest['entityCounts'] ?? [],
'modules' => $manifest['modules'] ?? [],
'sizeFormatted' => $this->formatBytes($manifest['totalAttachmentSizeBytes'] ?? 0),
];
}
return ResponseComposer::json(['backups' => $backups]);
}
private function formatBytes(int $bytes): string
{
if ($bytes >= 1073741824) {
return round($bytes / 1073741824, 2) . ' GB';
}
if ($bytes >= 1048576) {
return round($bytes / 1048576, 2) . ' MB';
}
if ($bytes >= 1024) {
return round($bytes / 1024, 2) . ' KB';
}
return $bytes . ' B';
}
}
@@ -0,0 +1,122 @@
<?php
namespace Espo\Modules\DataMigration\Api;
use Espo\Core\Api\Action;
use Espo\Core\Api\Request;
use Espo\Core\Api\Response;
use Espo\Core\Api\ResponseComposer;
use Espo\Core\Exceptions\Forbidden;
use Espo\Entities\User;
use Espo\ORM\EntityManager;
use Espo\Modules\DataMigration\Services\ManifestBuilder;
class Status implements Action
{
private const ENTITY_TYPES = [
'Case',
'Account',
'Contact',
'CaseActivity',
'Charge',
'Invoice',
'Document',
'NhMeeting',
'NhDecision',
'NhProcess',
'NhPlea',
'NhActivity',
'NhDocument',
'NhSyncLog',
'CaseMemory',
'SignatureRequest',
'SignatureRequestSigner',
'SmsLog',
'PricingAgreement',
'PricingAgreementRate',
'FixedActivityRate',
'ClientRate',
];
public function __construct(
private EntityManager $entityManager,
private ManifestBuilder $manifestBuilder,
private User $user
) {}
public function process(Request $request): Response
{
if (!$this->user->isAdmin()) {
throw new Forbidden();
}
$entityCounts = [];
$total = 0;
foreach (self::ENTITY_TYPES as $entityType) {
try {
$count = $this->entityManager
->getRDBRepository($entityType)
->count();
$entityCounts[$entityType] = $count;
$total += $count;
} catch (\Exception $e) {
// Entity type doesn't exist
}
}
$attachmentInfo = $this->getAttachmentInfo();
return ResponseComposer::json([
'entityCounts' => $entityCounts,
'totalRecords' => $total,
'attachmentFiles' => $attachmentInfo['count'],
'attachmentSize' => $attachmentInfo['size'],
'attachmentSizeFormatted' => $this->formatBytes($attachmentInfo['size']),
]);
}
private function getAttachmentInfo(): array
{
$uploadDir = 'data/upload/';
$count = 0;
$size = 0;
if (is_dir($uploadDir)) {
$files = scandir($uploadDir);
foreach ($files as $file) {
if ($file === '.' || $file === '..') {
continue;
}
$path = $uploadDir . $file;
if (is_file($path)) {
$count++;
$size += filesize($path);
}
}
}
return ['count' => $count, 'size' => $size];
}
private function formatBytes(int $bytes): string
{
if ($bytes >= 1073741824) {
return round($bytes / 1073741824, 2) . ' GB';
}
if ($bytes >= 1048576) {
return round($bytes / 1048576, 2) . ' MB';
}
if ($bytes >= 1024) {
return round($bytes / 1024, 2) . ' KB';
}
return $bytes . ' B';
}
}
@@ -0,0 +1,8 @@
{
"labels": {
"Data Migration": "Data Migration"
},
"descriptions": {
"dataMigration": "Export and import case data between EspoCRM instances"
}
}
@@ -0,0 +1,47 @@
{
"labels": {
"Data Migration": "Data Migration",
"Database Status": "Database Status",
"Export / Backup": "Export / Backup",
"Import / Restore": "Import / Restore",
"Available Backups": "Available Backups",
"No backups found": "No backups found",
"Start Export": "Start Export",
"Import Now": "Import Now",
"Dry Run": "Dry Run",
"Export completed": "Export completed successfully",
"Import completed": "Import completed successfully",
"Dry run completed": "Dry run completed successfully",
"Export failed": "Export failed",
"Import failed": "Import failed",
"Select a backup first": "Please select a backup first",
"Failed to delete backup": "Failed to delete backup",
"Selected backup": "Selected backup",
"Import Options": "Import Options",
"Preserve original IDs": "Preserve original IDs",
"Skip duplicates": "Skip duplicates",
"Skip attachment files": "Skip attachment files",
"Filter by Case Status": "Filter by Case Status",
"Exporting...": "Exporting...",
"Importing...": "Importing...",
"Log": "Log",
"records": "records",
"files": "files",
"optional": "optional",
"Entity Type": "Entity Type",
"Count": "Count",
"Total": "Total",
"Attachments": "Attachments",
"Modules": "Modules"
},
"messages": {
"exportDescription": "Export all case-related data to a backup file. You can filter by case status or export everything.",
"exportConfirmation": "Are you sure you want to start the export? This may take a while depending on the amount of data.",
"importConfirmation": "Are you sure you want to import this backup? This will add data to the database. This action cannot be easily undone.",
"dryRunConfirmation": "Run a dry import? This will simulate the import without making any changes to the database.",
"deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
"noFilesHint": "faster export, without attachment files",
"preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
"skipDuplicatesHint": "skip records that already exist in the database"
}
}
@@ -0,0 +1,8 @@
{
"labels": {
"Data Migration": "העברת נתונים"
},
"descriptions": {
"dataMigration": "ייצוא וייבוא נתוני תיקים בין מערכות EspoCRM"
}
}
@@ -0,0 +1,47 @@
{
"labels": {
"Data Migration": "העברת נתונים",
"Database Status": "מצב מסד הנתונים",
"Export / Backup": "ייצוא / גיבוי",
"Import / Restore": "ייבוא / שחזור",
"Available Backups": "גיבויים זמינים",
"No backups found": "לא נמצאו גיבויים",
"Start Export": "התחל ייצוא",
"Import Now": "ייבוא עכשיו",
"Dry Run": "הרצת ניסיון",
"Export completed": "הייצוא הושלם בהצלחה",
"Import completed": "הייבוא הושלם בהצלחה",
"Dry run completed": "הרצת הניסיון הושלמה בהצלחה",
"Export failed": "הייצוא נכשל",
"Import failed": "הייבוא נכשל",
"Select a backup first": "נא לבחור גיבוי קודם",
"Failed to delete backup": "מחיקת הגיבוי נכשלה",
"Selected backup": "גיבוי נבחר",
"Import Options": "אפשרויות ייבוא",
"Preserve original IDs": "שמור מזהים מקוריים",
"Skip duplicates": "דלג על כפילויות",
"Skip attachment files": "דלג על קבצים מצורפים",
"Filter by Case Status": "סנן לפי סטטוס תיק",
"Exporting...": "מייצא...",
"Importing...": "מייבא...",
"Log": "לוג",
"records": "רשומות",
"files": "קבצים",
"optional": "אופציונלי",
"Entity Type": "סוג ישות",
"Count": "כמות",
"Total": "סה\"כ",
"Attachments": "קבצים מצורפים",
"Modules": "מודולים"
},
"messages": {
"exportDescription": "ייצוא כל הנתונים הקשורים לתיקים לקובץ גיבוי. ניתן לסנן לפי סטטוס תיק או לייצא הכל.",
"exportConfirmation": "האם אתה בטוח שברצונך להתחיל את הייצוא? פעולה זו עשויה לקחת זמן בהתאם לכמות הנתונים.",
"importConfirmation": "האם אתה בטוח שברצונך לייבא גיבוי זה? פעולה זו תוסיף נתונים למסד הנתונים ולא ניתן לבטלה בקלות.",
"dryRunConfirmation": "להריץ ייבוא ניסיון? פעולה זו תדמה את הייבוא מבלי לבצע שינויים במסד הנתונים.",
"deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
"noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
"preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
"skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
}
}
@@ -0,0 +1,14 @@
{
"system": {
"itemList": [
{
"url": "#Admin/dataMigration",
"label": "Data Migration",
"iconClass": "fas fa-exchange-alt",
"description": "dataMigration",
"view": "modules/data-migration/views/admin/data-migration",
"order": 85
}
]
}
}
@@ -0,0 +1,27 @@
[
{
"route": "/DataMigration/action/status",
"method": "get",
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Status"
},
{
"route": "/DataMigration/action/export",
"method": "post",
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Export"
},
{
"route": "/DataMigration/action/import",
"method": "post",
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Import"
},
{
"route": "/DataMigration/action/listBackups",
"method": "get",
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\ListBackups"
},
{
"route": "/DataMigration/action/deleteBackup",
"method": "post",
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
}
]
@@ -0,0 +1,43 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\Core\Console\IO;
/**
* A buffered IO implementation that captures output lines
* for returning via API responses instead of console.
*/
class BufferedIO extends IO
{
/** @var string[] */
private array $lines = [];
public function write(string $string): void
{
$this->lines[] = $string;
}
public function writeLine(string $string): void
{
$this->lines[] = $string;
}
public function readLine(): string
{
return '';
}
/**
* @return string[]
*/
public function getLines(): array
{
return $this->lines;
}
public function getOutput(): string
{
return implode("\n", $this->lines);
}
}