feat: add admin UI for export/import data migration
- Admin panel page at #Admin/dataMigration with two-column layout - Export: filter by case status, skip attachments option, progress + log - Import: select backup, preserve IDs / skip duplicates, dry run support - List/delete existing backups with full manifest details - 5 API endpoints (status, export, import, listBackups, deleteBackup) - BufferedIO adapter to capture CLI output for API responses - Admin-only access on all endpoints - i18n: English + Hebrew (fa_IR) translations Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Api;
|
||||
|
||||
use Espo\Core\Api\Action;
|
||||
use Espo\Core\Api\Request;
|
||||
use Espo\Core\Api\Response;
|
||||
use Espo\Core\Api\ResponseComposer;
|
||||
use Espo\Core\Exceptions\BadRequest;
|
||||
use Espo\Core\Exceptions\Forbidden;
|
||||
use Espo\Entities\User;
|
||||
|
||||
class DeleteBackup implements Action
|
||||
{
|
||||
public function __construct(
|
||||
private User $user
|
||||
) {}
|
||||
|
||||
public function process(Request $request): Response
|
||||
{
|
||||
if (!$this->user->isAdmin()) {
|
||||
throw new Forbidden();
|
||||
}
|
||||
|
||||
$body = $request->getParsedBody();
|
||||
$backupName = $body->backupName ?? null;
|
||||
|
||||
if (!$backupName) {
|
||||
throw new BadRequest('backupName is required.');
|
||||
}
|
||||
|
||||
// Sanitize to prevent directory traversal
|
||||
$safeName = basename($backupName);
|
||||
|
||||
if (!str_starts_with($safeName, 'datamigration-')) {
|
||||
throw new Forbidden('Invalid backup name.');
|
||||
}
|
||||
|
||||
$fullPath = 'data/backups/' . $safeName;
|
||||
|
||||
if (!is_dir($fullPath)) {
|
||||
throw new BadRequest("Backup not found: {$safeName}");
|
||||
}
|
||||
|
||||
$this->deleteDirectory($fullPath);
|
||||
|
||||
return ResponseComposer::json([
|
||||
'success' => true,
|
||||
'deleted' => $safeName,
|
||||
]);
|
||||
}
|
||||
|
||||
private function deleteDirectory(string $dir): void
|
||||
{
|
||||
$items = scandir($dir);
|
||||
|
||||
foreach ($items as $item) {
|
||||
if ($item === '.' || $item === '..') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$path = $dir . '/' . $item;
|
||||
|
||||
if (is_dir($path)) {
|
||||
$this->deleteDirectory($path);
|
||||
} else {
|
||||
unlink($path);
|
||||
}
|
||||
}
|
||||
|
||||
rmdir($dir);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Api;
|
||||
|
||||
use Espo\Core\Api\Action;
|
||||
use Espo\Core\Api\Request;
|
||||
use Espo\Core\Api\Response;
|
||||
use Espo\Core\Api\ResponseComposer;
|
||||
use Espo\Core\Exceptions\Forbidden;
|
||||
use Espo\Entities\User;
|
||||
use Espo\Modules\DataMigration\Services\ExportService;
|
||||
use Espo\Modules\DataMigration\Services\BufferedIO;
|
||||
|
||||
class Export implements Action
|
||||
{
|
||||
public function __construct(
|
||||
private ExportService $exportService,
|
||||
private User $user
|
||||
) {}
|
||||
|
||||
public function process(Request $request): Response
|
||||
{
|
||||
if (!$this->user->isAdmin()) {
|
||||
throw new Forbidden();
|
||||
}
|
||||
|
||||
$body = $request->getParsedBody();
|
||||
|
||||
$caseIds = $body->caseIds ?? null;
|
||||
$caseStatus = $body->caseStatus ?? null;
|
||||
$noFiles = $body->noFiles ?? false;
|
||||
|
||||
$timestamp = date('Ymd-His');
|
||||
$outputDir = 'data/backups/datamigration-' . $timestamp;
|
||||
|
||||
$io = new BufferedIO();
|
||||
|
||||
$this->exportService->export($outputDir, [
|
||||
'batchSize' => 500,
|
||||
'caseIds' => $caseIds,
|
||||
'caseStatus' => is_array($caseStatus) && !empty($caseStatus) ? $caseStatus : null,
|
||||
'noFiles' => $noFiles,
|
||||
'verbose' => true,
|
||||
], $io);
|
||||
|
||||
return ResponseComposer::json([
|
||||
'success' => true,
|
||||
'outputDir' => $outputDir,
|
||||
'backupName' => 'datamigration-' . $timestamp,
|
||||
'log' => $io->getLines(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Api;
|
||||
|
||||
use Espo\Core\Api\Action;
|
||||
use Espo\Core\Api\Request;
|
||||
use Espo\Core\Api\Response;
|
||||
use Espo\Core\Api\ResponseComposer;
|
||||
use Espo\Core\Exceptions\BadRequest;
|
||||
use Espo\Core\Exceptions\Forbidden;
|
||||
use Espo\Entities\User;
|
||||
use Espo\Modules\DataMigration\Services\ImportService;
|
||||
use Espo\Modules\DataMigration\Services\BufferedIO;
|
||||
|
||||
class Import implements Action
|
||||
{
|
||||
public function __construct(
|
||||
private ImportService $importService,
|
||||
private User $user
|
||||
) {}
|
||||
|
||||
public function process(Request $request): Response
|
||||
{
|
||||
if (!$this->user->isAdmin()) {
|
||||
throw new Forbidden();
|
||||
}
|
||||
|
||||
$body = $request->getParsedBody();
|
||||
|
||||
$backupName = $body->backupName ?? null;
|
||||
|
||||
if (!$backupName) {
|
||||
throw new BadRequest('backupName is required.');
|
||||
}
|
||||
|
||||
$inputDir = 'data/backups/' . basename($backupName);
|
||||
|
||||
if (!is_dir($inputDir)) {
|
||||
throw new BadRequest("Backup not found: {$backupName}");
|
||||
}
|
||||
|
||||
if (!file_exists($inputDir . '/manifest.json')) {
|
||||
throw new BadRequest("Invalid backup: no manifest.json found.");
|
||||
}
|
||||
|
||||
$preserveIds = $body->preserveIds ?? true;
|
||||
$skipDuplicates = $body->skipDuplicates ?? true;
|
||||
$dryRun = $body->dryRun ?? false;
|
||||
|
||||
$io = new BufferedIO();
|
||||
|
||||
$this->importService->import($inputDir, [
|
||||
'batchSize' => 500,
|
||||
'preserveIds' => $preserveIds,
|
||||
'skipDuplicates' => $skipDuplicates,
|
||||
'dryRun' => $dryRun,
|
||||
'verbose' => true,
|
||||
], $io);
|
||||
|
||||
return ResponseComposer::json([
|
||||
'success' => true,
|
||||
'backupName' => $backupName,
|
||||
'dryRun' => $dryRun,
|
||||
'log' => $io->getLines(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Api;
|
||||
|
||||
use Espo\Core\Api\Action;
|
||||
use Espo\Core\Api\Request;
|
||||
use Espo\Core\Api\Response;
|
||||
use Espo\Core\Api\ResponseComposer;
|
||||
use Espo\Core\Exceptions\Forbidden;
|
||||
use Espo\Entities\User;
|
||||
use Espo\Modules\DataMigration\Services\ManifestBuilder;
|
||||
|
||||
class ListBackups implements Action
|
||||
{
|
||||
public function __construct(
|
||||
private ManifestBuilder $manifestBuilder,
|
||||
private User $user
|
||||
) {}
|
||||
|
||||
public function process(Request $request): Response
|
||||
{
|
||||
if (!$this->user->isAdmin()) {
|
||||
throw new Forbidden();
|
||||
}
|
||||
|
||||
$backupsDir = 'data/backups';
|
||||
$backups = [];
|
||||
|
||||
if (!is_dir($backupsDir)) {
|
||||
return ResponseComposer::json(['backups' => []]);
|
||||
}
|
||||
|
||||
$dirs = scandir($backupsDir, SCANDIR_SORT_DESCENDING);
|
||||
|
||||
foreach ($dirs as $dir) {
|
||||
if ($dir === '.' || $dir === '..') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!str_starts_with($dir, 'datamigration-')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$fullPath = $backupsDir . '/' . $dir;
|
||||
|
||||
if (!is_dir($fullPath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$manifest = $this->manifestBuilder->read($fullPath);
|
||||
|
||||
if ($manifest === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$totalRecords = 0;
|
||||
|
||||
foreach ($manifest['entityCounts'] ?? [] as $count) {
|
||||
$totalRecords += $count;
|
||||
}
|
||||
|
||||
$backups[] = [
|
||||
'name' => $dir,
|
||||
'exportedAt' => $manifest['exportedAt'] ?? null,
|
||||
'sourceHost' => $manifest['sourceHost'] ?? 'unknown',
|
||||
'espoVersion' => $manifest['espoVersion'] ?? 'unknown',
|
||||
'totalRecords' => $totalRecords,
|
||||
'totalAttachmentFiles' => $manifest['totalAttachmentFiles'] ?? 0,
|
||||
'totalAttachmentSizeBytes' => $manifest['totalAttachmentSizeBytes'] ?? 0,
|
||||
'entityCounts' => $manifest['entityCounts'] ?? [],
|
||||
'modules' => $manifest['modules'] ?? [],
|
||||
'sizeFormatted' => $this->formatBytes($manifest['totalAttachmentSizeBytes'] ?? 0),
|
||||
];
|
||||
}
|
||||
|
||||
return ResponseComposer::json(['backups' => $backups]);
|
||||
}
|
||||
|
||||
private function formatBytes(int $bytes): string
|
||||
{
|
||||
if ($bytes >= 1073741824) {
|
||||
return round($bytes / 1073741824, 2) . ' GB';
|
||||
}
|
||||
|
||||
if ($bytes >= 1048576) {
|
||||
return round($bytes / 1048576, 2) . ' MB';
|
||||
}
|
||||
|
||||
if ($bytes >= 1024) {
|
||||
return round($bytes / 1024, 2) . ' KB';
|
||||
}
|
||||
|
||||
return $bytes . ' B';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Api;
|
||||
|
||||
use Espo\Core\Api\Action;
|
||||
use Espo\Core\Api\Request;
|
||||
use Espo\Core\Api\Response;
|
||||
use Espo\Core\Api\ResponseComposer;
|
||||
use Espo\Core\Exceptions\Forbidden;
|
||||
use Espo\Entities\User;
|
||||
use Espo\ORM\EntityManager;
|
||||
use Espo\Modules\DataMigration\Services\ManifestBuilder;
|
||||
|
||||
class Status implements Action
|
||||
{
|
||||
private const ENTITY_TYPES = [
|
||||
'Case',
|
||||
'Account',
|
||||
'Contact',
|
||||
'CaseActivity',
|
||||
'Charge',
|
||||
'Invoice',
|
||||
'Document',
|
||||
'NhMeeting',
|
||||
'NhDecision',
|
||||
'NhProcess',
|
||||
'NhPlea',
|
||||
'NhActivity',
|
||||
'NhDocument',
|
||||
'NhSyncLog',
|
||||
'CaseMemory',
|
||||
'SignatureRequest',
|
||||
'SignatureRequestSigner',
|
||||
'SmsLog',
|
||||
'PricingAgreement',
|
||||
'PricingAgreementRate',
|
||||
'FixedActivityRate',
|
||||
'ClientRate',
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private EntityManager $entityManager,
|
||||
private ManifestBuilder $manifestBuilder,
|
||||
private User $user
|
||||
) {}
|
||||
|
||||
public function process(Request $request): Response
|
||||
{
|
||||
if (!$this->user->isAdmin()) {
|
||||
throw new Forbidden();
|
||||
}
|
||||
|
||||
$entityCounts = [];
|
||||
$total = 0;
|
||||
|
||||
foreach (self::ENTITY_TYPES as $entityType) {
|
||||
try {
|
||||
$count = $this->entityManager
|
||||
->getRDBRepository($entityType)
|
||||
->count();
|
||||
|
||||
$entityCounts[$entityType] = $count;
|
||||
$total += $count;
|
||||
} catch (\Exception $e) {
|
||||
// Entity type doesn't exist
|
||||
}
|
||||
}
|
||||
|
||||
$attachmentInfo = $this->getAttachmentInfo();
|
||||
|
||||
return ResponseComposer::json([
|
||||
'entityCounts' => $entityCounts,
|
||||
'totalRecords' => $total,
|
||||
'attachmentFiles' => $attachmentInfo['count'],
|
||||
'attachmentSize' => $attachmentInfo['size'],
|
||||
'attachmentSizeFormatted' => $this->formatBytes($attachmentInfo['size']),
|
||||
]);
|
||||
}
|
||||
|
||||
private function getAttachmentInfo(): array
|
||||
{
|
||||
$uploadDir = 'data/upload/';
|
||||
$count = 0;
|
||||
$size = 0;
|
||||
|
||||
if (is_dir($uploadDir)) {
|
||||
$files = scandir($uploadDir);
|
||||
|
||||
foreach ($files as $file) {
|
||||
if ($file === '.' || $file === '..') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$path = $uploadDir . $file;
|
||||
|
||||
if (is_file($path)) {
|
||||
$count++;
|
||||
$size += filesize($path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ['count' => $count, 'size' => $size];
|
||||
}
|
||||
|
||||
private function formatBytes(int $bytes): string
|
||||
{
|
||||
if ($bytes >= 1073741824) {
|
||||
return round($bytes / 1073741824, 2) . ' GB';
|
||||
}
|
||||
|
||||
if ($bytes >= 1048576) {
|
||||
return round($bytes / 1048576, 2) . ' MB';
|
||||
}
|
||||
|
||||
if ($bytes >= 1024) {
|
||||
return round($bytes / 1024, 2) . ' KB';
|
||||
}
|
||||
|
||||
return $bytes . ' B';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"labels": {
|
||||
"Data Migration": "Data Migration"
|
||||
},
|
||||
"descriptions": {
|
||||
"dataMigration": "Export and import case data between EspoCRM instances"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"labels": {
|
||||
"Data Migration": "Data Migration",
|
||||
"Database Status": "Database Status",
|
||||
"Export / Backup": "Export / Backup",
|
||||
"Import / Restore": "Import / Restore",
|
||||
"Available Backups": "Available Backups",
|
||||
"No backups found": "No backups found",
|
||||
"Start Export": "Start Export",
|
||||
"Import Now": "Import Now",
|
||||
"Dry Run": "Dry Run",
|
||||
"Export completed": "Export completed successfully",
|
||||
"Import completed": "Import completed successfully",
|
||||
"Dry run completed": "Dry run completed successfully",
|
||||
"Export failed": "Export failed",
|
||||
"Import failed": "Import failed",
|
||||
"Select a backup first": "Please select a backup first",
|
||||
"Failed to delete backup": "Failed to delete backup",
|
||||
"Selected backup": "Selected backup",
|
||||
"Import Options": "Import Options",
|
||||
"Preserve original IDs": "Preserve original IDs",
|
||||
"Skip duplicates": "Skip duplicates",
|
||||
"Skip attachment files": "Skip attachment files",
|
||||
"Filter by Case Status": "Filter by Case Status",
|
||||
"Exporting...": "Exporting...",
|
||||
"Importing...": "Importing...",
|
||||
"Log": "Log",
|
||||
"records": "records",
|
||||
"files": "files",
|
||||
"optional": "optional",
|
||||
"Entity Type": "Entity Type",
|
||||
"Count": "Count",
|
||||
"Total": "Total",
|
||||
"Attachments": "Attachments",
|
||||
"Modules": "Modules"
|
||||
},
|
||||
"messages": {
|
||||
"exportDescription": "Export all case-related data to a backup file. You can filter by case status or export everything.",
|
||||
"exportConfirmation": "Are you sure you want to start the export? This may take a while depending on the amount of data.",
|
||||
"importConfirmation": "Are you sure you want to import this backup? This will add data to the database. This action cannot be easily undone.",
|
||||
"dryRunConfirmation": "Run a dry import? This will simulate the import without making any changes to the database.",
|
||||
"deleteBackupConfirmation": "Are you sure you want to delete this backup? This action cannot be undone.",
|
||||
"noFilesHint": "faster export, without attachment files",
|
||||
"preserveIdsHint": "keep original record IDs (recommended for same-instance restore)",
|
||||
"skipDuplicatesHint": "skip records that already exist in the database"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"labels": {
|
||||
"Data Migration": "העברת נתונים"
|
||||
},
|
||||
"descriptions": {
|
||||
"dataMigration": "ייצוא וייבוא נתוני תיקים בין מערכות EspoCRM"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"labels": {
|
||||
"Data Migration": "העברת נתונים",
|
||||
"Database Status": "מצב מסד הנתונים",
|
||||
"Export / Backup": "ייצוא / גיבוי",
|
||||
"Import / Restore": "ייבוא / שחזור",
|
||||
"Available Backups": "גיבויים זמינים",
|
||||
"No backups found": "לא נמצאו גיבויים",
|
||||
"Start Export": "התחל ייצוא",
|
||||
"Import Now": "ייבוא עכשיו",
|
||||
"Dry Run": "הרצת ניסיון",
|
||||
"Export completed": "הייצוא הושלם בהצלחה",
|
||||
"Import completed": "הייבוא הושלם בהצלחה",
|
||||
"Dry run completed": "הרצת הניסיון הושלמה בהצלחה",
|
||||
"Export failed": "הייצוא נכשל",
|
||||
"Import failed": "הייבוא נכשל",
|
||||
"Select a backup first": "נא לבחור גיבוי קודם",
|
||||
"Failed to delete backup": "מחיקת הגיבוי נכשלה",
|
||||
"Selected backup": "גיבוי נבחר",
|
||||
"Import Options": "אפשרויות ייבוא",
|
||||
"Preserve original IDs": "שמור מזהים מקוריים",
|
||||
"Skip duplicates": "דלג על כפילויות",
|
||||
"Skip attachment files": "דלג על קבצים מצורפים",
|
||||
"Filter by Case Status": "סנן לפי סטטוס תיק",
|
||||
"Exporting...": "מייצא...",
|
||||
"Importing...": "מייבא...",
|
||||
"Log": "לוג",
|
||||
"records": "רשומות",
|
||||
"files": "קבצים",
|
||||
"optional": "אופציונלי",
|
||||
"Entity Type": "סוג ישות",
|
||||
"Count": "כמות",
|
||||
"Total": "סה\"כ",
|
||||
"Attachments": "קבצים מצורפים",
|
||||
"Modules": "מודולים"
|
||||
},
|
||||
"messages": {
|
||||
"exportDescription": "ייצוא כל הנתונים הקשורים לתיקים לקובץ גיבוי. ניתן לסנן לפי סטטוס תיק או לייצא הכל.",
|
||||
"exportConfirmation": "האם אתה בטוח שברצונך להתחיל את הייצוא? פעולה זו עשויה לקחת זמן בהתאם לכמות הנתונים.",
|
||||
"importConfirmation": "האם אתה בטוח שברצונך לייבא גיבוי זה? פעולה זו תוסיף נתונים למסד הנתונים ולא ניתן לבטלה בקלות.",
|
||||
"dryRunConfirmation": "להריץ ייבוא ניסיון? פעולה זו תדמה את הייבוא מבלי לבצע שינויים במסד הנתונים.",
|
||||
"deleteBackupConfirmation": "האם אתה בטוח שברצונך למחוק גיבוי זה? לא ניתן לבטל פעולה זו.",
|
||||
"noFilesHint": "ייצוא מהיר יותר, ללא קבצים מצורפים",
|
||||
"preserveIdsHint": "שמור מזהי רשומות מקוריים (מומלץ לשחזור באותה מערכת)",
|
||||
"skipDuplicatesHint": "דלג על רשומות שכבר קיימות במסד הנתונים"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"system": {
|
||||
"itemList": [
|
||||
{
|
||||
"url": "#Admin/dataMigration",
|
||||
"label": "Data Migration",
|
||||
"iconClass": "fas fa-exchange-alt",
|
||||
"description": "dataMigration",
|
||||
"view": "modules/data-migration/views/admin/data-migration",
|
||||
"order": 85
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
[
|
||||
{
|
||||
"route": "/DataMigration/action/status",
|
||||
"method": "get",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Status"
|
||||
},
|
||||
{
|
||||
"route": "/DataMigration/action/export",
|
||||
"method": "post",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Export"
|
||||
},
|
||||
{
|
||||
"route": "/DataMigration/action/import",
|
||||
"method": "post",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\Import"
|
||||
},
|
||||
{
|
||||
"route": "/DataMigration/action/listBackups",
|
||||
"method": "get",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\ListBackups"
|
||||
},
|
||||
{
|
||||
"route": "/DataMigration/action/deleteBackup",
|
||||
"method": "post",
|
||||
"actionClassName": "Espo\\Modules\\DataMigration\\Api\\DeleteBackup"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
namespace Espo\Modules\DataMigration\Services;
|
||||
|
||||
use Espo\Core\Console\IO;
|
||||
|
||||
/**
|
||||
* A buffered IO implementation that captures output lines
|
||||
* for returning via API responses instead of console.
|
||||
*/
|
||||
class BufferedIO extends IO
|
||||
{
|
||||
/** @var string[] */
|
||||
private array $lines = [];
|
||||
|
||||
public function write(string $string): void
|
||||
{
|
||||
$this->lines[] = $string;
|
||||
}
|
||||
|
||||
public function writeLine(string $string): void
|
||||
{
|
||||
$this->lines[] = $string;
|
||||
}
|
||||
|
||||
public function readLine(): string
|
||||
{
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string[]
|
||||
*/
|
||||
public function getLines(): array
|
||||
{
|
||||
return $this->lines;
|
||||
}
|
||||
|
||||
public function getOutput(): string
|
||||
{
|
||||
return implode("\n", $this->lines);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user