feat: add admin UI for export/import data migration

- Admin panel page at #Admin/dataMigration with two-column layout
- Export: filter by case status, skip attachments option, progress + log
- Import: select backup, preserve IDs / skip duplicates, dry run support
- List/delete existing backups with full manifest details
- 5 API endpoints (status, export, import, listBackups, deleteBackup)
- BufferedIO adapter to capture CLI output for API responses
- Admin-only access on all endpoints
- i18n: English + Hebrew (fa_IR) translations

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-03-29 19:45:20 +00:00
parent e08eca7eb3
commit 6d6f5d571a
15 changed files with 1211 additions and 0 deletions
@@ -0,0 +1,179 @@
<div class="page-header">
<h3>
<a href="#Admin">{{translate 'Administration'}}</a>
<span class="breadcrumb-separator"><span class="chevron-right"></span></span>
{{translate 'Data Migration' scope='DataMigration'}}
</h3>
</div>
<div class="row">
{{! ========== LEFT COLUMN: Status + Export ========== }}
<div class="col-md-6">
{{! === Database Status === }}
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
<button class="btn btn-default btn-xs" data-action="refreshStatus" title="{{translate 'Refresh'}}">
<i class="fas fa-sync-alt"></i>
</button>
</div>
<h4 class="panel-title">
<i class="fas fa-database"></i>
{{translate 'Database Status' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body">
<div class="status-spinner" style="text-align: center; padding: 20px;">
<i class="fas fa-spinner fa-spin fa-2x text-muted"></i>
</div>
<div class="status-content" style="display: none;"></div>
</div>
</div>
{{! === Export === }}
<div class="panel panel-primary">
<div class="panel-heading">
<h4 class="panel-title">
<i class="fas fa-upload"></i>
{{translate 'Export / Backup' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body">
<p class="text-muted" style="margin-bottom: 15px;">
{{translate 'exportDescription' category='messages' scope='DataMigration'}}
</p>
{{! Status filter }}
<div class="form-group">
<label class="control-label">
{{translate 'Filter by Case Status' scope='DataMigration'}}
<span class="text-muted" style="font-weight: normal;"> ({{translate 'optional' scope='DataMigration'}})</span>
</label>
<div class="export-status-checkboxes" style="columns: 2;">
{{#each caseStatuses}}
<div class="checkbox" style="margin-top: 2px; margin-bottom: 2px;">
<label>
<input type="checkbox" value="{{this}}"> {{this}}
</label>
</div>
{{/each}}
</div>
</div>
{{! No files option }}
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" name="noFiles">
{{translate 'Skip attachment files' scope='DataMigration'}}
<span class="text-muted"> — {{translate 'noFilesHint' category='messages' scope='DataMigration'}}</span>
</label>
</div>
</div>
<div>
<button class="btn btn-primary" data-action="export">
<i class="fas fa-download"></i>
{{translate 'Start Export' scope='DataMigration'}}
</button>
</div>
<div class="export-progress" style="display: none; margin-top: 15px;">
<div class="progress">
<div class="progress-bar progress-bar-striped active" style="width: 100%;">
{{translate 'Exporting...' scope='DataMigration'}}
</div>
</div>
</div>
<div class="export-log-container" style="display: none;"></div>
</div>
</div>
</div>
{{! ========== RIGHT COLUMN: Backups + Import ========== }}
<div class="col-md-6">
{{! === Backup List === }}
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
<button class="btn btn-default btn-xs" data-action="refreshBackups" title="{{translate 'Refresh'}}">
<i class="fas fa-sync-alt"></i>
</button>
</div>
<h4 class="panel-title">
<i class="fas fa-archive"></i>
{{translate 'Available Backups' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body" style="padding: 0;">
<div class="backups-spinner" style="text-align: center; padding: 20px;">
<i class="fas fa-spinner fa-spin fa-2x text-muted"></i>
</div>
<div class="backups-list list-group" style="margin-bottom: 0;"></div>
</div>
</div>
{{! === Import === }}
<div class="import-section" style="display: none;">
<div class="panel panel-warning">
<div class="panel-heading">
<h4 class="panel-title">
<i class="fas fa-download"></i>
{{translate 'Import / Restore' scope='DataMigration'}}
</h4>
</div>
<div class="panel-body">
<div class="alert alert-info" style="margin-bottom: 15px;">
<i class="fas fa-info-circle"></i>
{{translate 'Selected backup' scope='DataMigration'}}: <strong class="selected-backup-name"></strong>
</div>
{{! Import options }}
<div class="form-group">
<label class="control-label">{{translate 'Import Options' scope='DataMigration'}}</label>
<div class="checkbox">
<label>
<input type="checkbox" name="preserveIds" checked>
{{translate 'Preserve original IDs' scope='DataMigration'}}
<span class="text-muted"> — {{translate 'preserveIdsHint' category='messages' scope='DataMigration'}}</span>
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="skipDuplicates" checked>
{{translate 'Skip duplicates' scope='DataMigration'}}
<span class="text-muted"> — {{translate 'skipDuplicatesHint' category='messages' scope='DataMigration'}}</span>
</label>
</div>
</div>
<div class="btn-group">
<button class="btn btn-warning" data-action="dryRunImport">
<i class="fas fa-vial"></i>
{{translate 'Dry Run' scope='DataMigration'}}
</button>
<button class="btn btn-danger" data-action="import">
<i class="fas fa-play"></i>
{{translate 'Import Now' scope='DataMigration'}}
</button>
</div>
<div class="import-progress" style="display: none; margin-top: 15px;">
<div class="progress">
<div class="progress-bar progress-bar-warning progress-bar-striped active" style="width: 100%;">
{{translate 'Importing...' scope='DataMigration'}}
</div>
</div>
</div>
<div class="import-log-container" style="display: none;"></div>
</div>
</div>
</div>
</div>
</div>
@@ -0,0 +1,427 @@
define('modules/data-migration/views/admin/data-migration', ['view'], function (Dep) {
const Parent = Dep.__esModule ? Dep.default : Dep;
return Parent.extend({
template: 'data-migration:admin/data-migration',
isExporting: false,
isImporting: false,
isLoadingStatus: false,
isLoadingBackups: false,
data: function () {
return {
isExporting: this.isExporting,
isImporting: this.isImporting,
isLoadingStatus: this.isLoadingStatus,
statusData: this.statusData,
backups: this.backups,
selectedBackup: this.selectedBackup,
exportLog: this.exportLog,
importLog: this.importLog,
caseStatuses: this.caseStatuses,
};
},
setup: function () {
this.statusData = null;
this.backups = [];
this.selectedBackup = null;
this.exportLog = null;
this.importLog = null;
this.caseStatuses = [
'Active', 'Pending', 'Closed', 'Rejected',
'On Hold', 'Won', 'Lost', 'Archived',
];
this.once('after:render', () => {
this.loadStatus();
this.loadBackups();
});
},
events: {
'click [data-action="export"]': function () {
this.actionExport();
},
'click [data-action="import"]': function () {
this.actionImport();
},
'click [data-action="dryRunImport"]': function () {
this.actionImport(true);
},
'click [data-action="refreshStatus"]': function () {
this.loadStatus();
},
'click [data-action="refreshBackups"]': function () {
this.loadBackups();
},
'click [data-action="selectBackup"]': function (e) {
const name = $(e.currentTarget).data('name');
this.selectBackup(name);
},
'click [data-action="deleteBackup"]': function (e) {
e.stopPropagation();
const name = $(e.currentTarget).data('name');
this.actionDeleteBackup(name);
},
'click [data-action="toggleBackupDetails"]': function (e) {
const name = $(e.currentTarget).data('name');
this.$el.find('.backup-details[data-name="' + name + '"]').toggle();
},
'click [data-action="dismissExportLog"]': function () {
this.exportLog = null;
this.$el.find('.export-log-container').hide();
},
'click [data-action="dismissImportLog"]': function () {
this.importLog = null;
this.$el.find('.import-log-container').hide();
},
},
loadStatus: function () {
this.isLoadingStatus = true;
this.$el.find('.status-spinner').show();
this.$el.find('.status-content').hide();
Espo.Ajax.getRequest('DataMigration/action/status')
.then(response => {
this.statusData = response;
this.isLoadingStatus = false;
this.renderStatus();
})
.catch(() => {
this.isLoadingStatus = false;
this.$el.find('.status-spinner').hide();
Espo.Ui.error('Failed to load status');
});
},
renderStatus: function () {
const $container = this.$el.find('.status-content');
const $spinner = this.$el.find('.status-spinner');
$spinner.hide();
if (!this.statusData) {
$container.html('<p class="text-muted">No data</p>').show();
return;
}
let html = '<table class="table table-bordered table-sm">';
html += '<thead><tr><th>' + this.translate('Entity Type') + '</th><th class="text-right">' + this.translate('Count') + '</th></tr></thead>';
html += '<tbody>';
let total = 0;
const counts = this.statusData.entityCounts || {};
for (const type in counts) {
if (counts[type] > 0) {
html += '<tr><td>' + type + '</td><td class="text-right">' + counts[type].toLocaleString() + '</td></tr>';
total += counts[type];
}
}
html += '<tr class="active"><td><strong>' + this.translate('Total') + '</strong></td>';
html += '<td class="text-right"><strong>' + total.toLocaleString() + '</strong></td></tr>';
html += '</tbody></table>';
html += '<p class="text-muted" style="margin-top: 8px;">';
html += '<i class="fas fa-paperclip"></i> ';
html += this.translate('Attachments') + ': ' + (this.statusData.attachmentFiles || 0).toLocaleString();
html += ' (' + (this.statusData.attachmentSizeFormatted || '0 B') + ')';
html += '</p>';
$container.html(html).show();
},
loadBackups: function () {
this.isLoadingBackups = true;
this.$el.find('.backups-spinner').show();
Espo.Ajax.getRequest('DataMigration/action/listBackups')
.then(response => {
this.backups = response.backups || [];
this.isLoadingBackups = false;
this.renderBackups();
})
.catch(() => {
this.isLoadingBackups = false;
this.$el.find('.backups-spinner').hide();
Espo.Ui.error('Failed to load backups');
});
},
renderBackups: function () {
const $container = this.$el.find('.backups-list');
const $spinner = this.$el.find('.backups-spinner');
$spinner.hide();
if (!this.backups || this.backups.length === 0) {
$container.html(
'<div class="no-data text-muted" style="padding: 20px; text-align: center;">' +
'<i class="fas fa-archive" style="font-size: 32px; margin-bottom: 10px; display: block;"></i>' +
this.translate('No backups found', 'labels', 'DataMigration') +
'</div>'
);
return;
}
let html = '';
this.backups.forEach(backup => {
const isSelected = this.selectedBackup === backup.name;
const selectedClass = isSelected ? ' active' : '';
const date = backup.exportedAt ? new Date(backup.exportedAt).toLocaleString() : 'Unknown';
let totalRecords = 0;
const ec = backup.entityCounts || {};
for (const k in ec) {
totalRecords += ec[k];
}
html += '<div class="backup-item list-group-item' + selectedClass + '" data-action="selectBackup" data-name="' + backup.name + '" style="cursor: pointer;">';
html += '<div class="pull-right">';
html += '<button class="btn btn-danger btn-xs" data-action="deleteBackup" data-name="' + backup.name + '" title="' + this.translate('Delete') + '">';
html += '<i class="fas fa-trash"></i></button>';
html += '</div>';
html += '<h5 class="list-group-item-heading" style="margin: 0 0 4px;">';
html += '<i class="fas fa-archive text-muted"></i> ';
html += '<span data-action="toggleBackupDetails" data-name="' + backup.name + '">' + backup.name + '</span>';
html += '</h5>';
html += '<p class="list-group-item-text text-muted" style="margin: 0; font-size: 0.9em;">';
html += '<i class="fas fa-clock"></i> ' + date;
html += ' &nbsp;|&nbsp; <i class="fas fa-database"></i> ' + totalRecords.toLocaleString() + ' ' + this.translate('records', 'labels', 'DataMigration');
html += ' &nbsp;|&nbsp; <i class="fas fa-paperclip"></i> ' + (backup.totalAttachmentFiles || 0).toLocaleString() + ' ' + this.translate('files', 'labels', 'DataMigration');
html += ' (' + (backup.sizeFormatted || '0 B') + ')';
html += '</p>';
// Details (hidden by default)
html += '<div class="backup-details" data-name="' + backup.name + '" style="display: none; margin-top: 10px;">';
html += '<table class="table table-condensed table-sm" style="margin-bottom: 0;">';
for (const et in ec) {
if (ec[et] > 0) {
html += '<tr><td style="padding: 2px 8px;">' + et + '</td><td style="padding: 2px 8px;" class="text-right">' + ec[et].toLocaleString() + '</td></tr>';
}
}
html += '</table>';
if (backup.modules && Object.keys(backup.modules).length > 0) {
html += '<div class="text-muted" style="margin-top: 5px; font-size: 0.85em;">';
html += '<strong>' + this.translate('Modules') + ':</strong> ';
const moduleParts = [];
for (const mod in backup.modules) {
moduleParts.push(mod + ' v' + backup.modules[mod]);
}
html += moduleParts.join(', ');
html += '</div>';
}
html += '</div>';
html += '</div>';
});
$container.html(html);
},
selectBackup: function (name) {
if (this.selectedBackup === name) {
this.selectedBackup = null;
} else {
this.selectedBackup = name;
}
this.$el.find('.backup-item').removeClass('active');
if (this.selectedBackup) {
this.$el.find('.backup-item[data-name="' + name + '"]').addClass('active');
this.$el.find('.import-section').show();
this.$el.find('.selected-backup-name').text(name);
} else {
this.$el.find('.import-section').hide();
}
},
actionExport: function () {
const statusFilters = [];
this.$el.find('.export-status-checkboxes input:checked').each(function () {
statusFilters.push($(this).val());
});
const noFiles = this.$el.find('input[name="noFiles"]').is(':checked');
this.confirm(
this.translate('exportConfirmation', 'messages', 'DataMigration'),
() => {
this.isExporting = true;
this.$el.find('[data-action="export"]').addClass('disabled').attr('disabled', true);
this.$el.find('.export-progress').show();
Espo.Ajax.postRequest('DataMigration/action/export', {
caseStatus: statusFilters.length > 0 ? statusFilters : null,
noFiles: noFiles,
}, {timeout: 600000})
.then(response => {
this.isExporting = false;
this.$el.find('[data-action="export"]').removeClass('disabled').attr('disabled', false);
this.$el.find('.export-progress').hide();
this.exportLog = response.log || [];
this.renderLog('.export-log-container', this.exportLog, true);
Espo.Ui.success(this.translate('Export completed', 'labels', 'DataMigration'));
this.loadBackups();
this.loadStatus();
})
.catch(xhr => {
this.isExporting = false;
this.$el.find('[data-action="export"]').removeClass('disabled').attr('disabled', false);
this.$el.find('.export-progress').hide();
Espo.Ui.error(this.translate('Export failed', 'labels', 'DataMigration'));
});
}
);
},
actionImport: function (dryRun) {
dryRun = dryRun || false;
if (!this.selectedBackup) {
Espo.Ui.warning(this.translate('Select a backup first', 'labels', 'DataMigration'));
return;
}
const preserveIds = this.$el.find('input[name="preserveIds"]').is(':checked');
const skipDuplicates = this.$el.find('input[name="skipDuplicates"]').is(':checked');
const confirmMsg = dryRun
? this.translate('dryRunConfirmation', 'messages', 'DataMigration')
: this.translate('importConfirmation', 'messages', 'DataMigration');
this.confirm(confirmMsg, () => {
this.isImporting = true;
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
.addClass('disabled').attr('disabled', true);
this.$el.find('.import-progress').show();
Espo.Ajax.postRequest('DataMigration/action/import', {
backupName: this.selectedBackup,
preserveIds: preserveIds,
skipDuplicates: skipDuplicates,
dryRun: dryRun,
}, {timeout: 600000})
.then(response => {
this.isImporting = false;
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
.removeClass('disabled').attr('disabled', false);
this.$el.find('.import-progress').hide();
this.importLog = response.log || [];
this.renderLog('.import-log-container', this.importLog, !dryRun);
const label = dryRun ? 'Dry run completed' : 'Import completed';
Espo.Ui.success(this.translate(label, 'labels', 'DataMigration'));
if (!dryRun) {
this.loadStatus();
}
})
.catch(xhr => {
this.isImporting = false;
this.$el.find('[data-action="import"], [data-action="dryRunImport"]')
.removeClass('disabled').attr('disabled', false);
this.$el.find('.import-progress').hide();
Espo.Ui.error(this.translate('Import failed', 'labels', 'DataMigration'));
});
});
},
actionDeleteBackup: function (name) {
this.confirm(
this.translate('deleteBackupConfirmation', 'messages', 'DataMigration'),
() => {
Espo.Ajax.postRequest('DataMigration/action/deleteBackup', {
backupName: name,
})
.then(() => {
if (this.selectedBackup === name) {
this.selectedBackup = null;
this.$el.find('.import-section').hide();
}
Espo.Ui.success(this.translate('Deleted'));
this.loadBackups();
})
.catch(() => {
Espo.Ui.error(this.translate('Failed to delete backup', 'labels', 'DataMigration'));
});
}
);
},
renderLog: function (selector, lines, isSuccess) {
const $container = this.$el.find(selector);
if (!lines || lines.length === 0) {
$container.hide();
return;
}
const panelClass = isSuccess ? 'panel-success' : 'panel-info';
const dismissAction = selector.includes('export') ? 'dismissExportLog' : 'dismissImportLog';
let html = '<div class="panel ' + panelClass + '" style="margin-top: 15px;">';
html += '<div class="panel-heading">';
html += '<button type="button" class="close" data-action="' + dismissAction + '">&times;</button>';
html += '<h4 class="panel-title"><i class="fas fa-terminal"></i> ' + this.translate('Log', 'labels', 'DataMigration') + '</h4>';
html += '</div>';
html += '<div class="panel-body" style="max-height: 400px; overflow-y: auto;">';
html += '<pre style="margin: 0; white-space: pre-wrap; font-size: 12px; background: transparent; border: none; padding: 0;">';
lines.forEach(line => {
let lineClass = '';
if (line.includes('ERROR')) {
lineClass = 'text-danger';
} else if (line.includes('WARNING')) {
lineClass = 'text-warning';
} else if (line.includes('===') || line.includes('Complete')) {
lineClass = 'text-success';
}
if (lineClass) {
html += '<span class="' + lineClass + '">' + this.escapeHtml(line) + '</span>\n';
} else {
html += this.escapeHtml(line) + '\n';
}
});
html += '</pre></div></div>';
$container.html(html).show();
},
escapeHtml: function (text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
},
confirm: function (message, callback) {
Espo.Ui.confirm(message, {
confirmText: this.translate('Yes'),
cancelText: this.translate('No'),
}, callback);
},
});
});