espocrm-extensions/DataMigration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 14 Wiki Activity

feat: initial DataMigration extension for case data backup/restore

Browse Source 
Full case data migration tool with 3 console commands:
- data-migration-export: exports all case-related entities (27 types) + files
- data-migration-import: restores backup to target DB with dry-run support
- data-migration-status: shows entity counts from live DB or backup

Supports preserve-ids, skip-duplicates, batch processing, and attachment
file copying. Imports with hooks disabled to prevent side effects.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chaim Marcus
2026-03-28 19:50:46 +00:00
commit e08eca7eb3
15 changed files with 2056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/bin/bash
# Build extension ZIP package
set -euo pipefail
VERSION=$(python3 -c "import json; print(json.load(open('manifest.json'))['version'])")
MODULE=$(python3 -c "import json; m=json.load(open('manifest.json')); print(m.get('module', m['name']))")
ZIPNAME="${MODULE}-${VERSION}.zip"
# Update version in README.md if it exists
if [[ -f README.md ]]; then
sed -i "s/\*\*גרסה:\*\* [^ |]*/\*\*גרסה:\*\* ${VERSION}/" README.md
fi
echo "Building $ZIPNAME..."
rm -f "$ZIPNAME"
zip -r "$ZIPNAME" manifest.json files/ scripts/ \
-x "*.DS_Store" "*__MACOSX*" "*.zip" 2>/dev/null || \
zip -r "$ZIPNAME" manifest.json files/ \
-x "*.DS_Store" "*__MACOSX*" "*.zip"
echo "✓ Built: $ZIPNAME ($(du -h "$ZIPNAME" | cut -f1))"
files/custom/Espo/Modules/DataMigration/Commands/DataMigrationExport.php
+78
View File
@@ -0,0 +1,78 @@
<?php
namespace Espo\Modules\DataMigration\Commands;
use Espo\Core\Console\Command;
use Espo\Core\Console\Command\Params;
use Espo\Core\Console\IO;
use Espo\Modules\DataMigration\Services\ExportService;
class DataMigrationExport implements Command
{
public function __construct(
private ExportService $exportService
) {}
public function run(Params $params, IO $io): void
{
$output = $params->getOption('output');
if (!$output) {
$timestamp = date('Ymd-His');
$output = 'data/backups/datamigration-' . $timestamp;
}
$output = rtrim($output, '/');
if (is_dir($output)) {
$io->writeLine("ERROR: Output directory already exists: {$output}");
$io->writeLine("Please specify a new directory or remove the existing one.");
return;
}
$batchSize = (int) ($params->getOption('batchSize') ?? 500);
$verbose = $params->hasFlag('verbose');
$noFiles = $params->hasFlag('noFiles');
$caseIds = null;
$caseIdsStr = $params->getOption('caseIds');
if ($caseIdsStr) {
$caseIds = array_map('trim', explode(',', $caseIdsStr));
}
$caseStatus = null;
$statusStr = $params->getOption('status');
if ($statusStr) {
$caseStatus = array_map('trim', explode(',', $statusStr));
}
$io->writeLine('EspoCRM Data Migration - Export');
$io->writeLine('==============================');
$io->writeLine("Output: {$output}");
$io->writeLine("Batch size: {$batchSize}");
if ($caseIds) {
$io->writeLine("Case filter: " . count($caseIds) . " specific IDs");
}
if ($caseStatus) {
$io->writeLine("Status filter: " . implode(', ', $caseStatus));
}
if ($noFiles) {
$io->writeLine("Files: SKIPPED (--no-files)");
}
$io->writeLine('');
$this->exportService->export($output, [
'batchSize' => $batchSize,
'caseIds' => $caseIds,
'caseStatus' => $caseStatus,
'noFiles' => $noFiles,
'verbose' => $verbose,
], $io);
}
}
files/custom/Espo/Modules/DataMigration/Commands/DataMigrationImport.php
+61
View File
@@ -0,0 +1,61 @@
<?php
namespace Espo\Modules\DataMigration\Commands;
use Espo\Core\Console\Command;
use Espo\Core\Console\Command\Params;
use Espo\Core\Console\IO;
use Espo\Modules\DataMigration\Services\ImportService;
class DataMigrationImport implements Command
{
public function __construct(
private ImportService $importService
) {}
public function run(Params $params, IO $io): void
{
$input = $params->getOption('input');
if (!$input) {
$io->writeLine('ERROR: --input option is required. Specify the backup directory path.');
return;
}
$input = rtrim($input, '/');
if (!is_dir($input)) {
$io->writeLine("ERROR: Input directory not found: {$input}");
return;
}
if (!file_exists($input . '/manifest.json')) {
$io->writeLine("ERROR: No manifest.json found in: {$input}");
$io->writeLine("This does not appear to be a valid backup directory.");
return;
}
$batchSize = (int) ($params->getOption('batchSize') ?? 500);
$dryRun = $params->hasFlag('dryRun');
$preserveIds = $params->hasFlag('preserveIds');
$skipDuplicates = $params->hasFlag('skipDuplicates');
$verbose = $params->hasFlag('verbose');
$io->writeLine('EspoCRM Data Migration - Import');
$io->writeLine('==============================');
$io->writeLine("Input: {$input}");
$io->writeLine("Batch size: {$batchSize}");
$io->writeLine("Preserve IDs: " . ($preserveIds ? 'YES' : 'NO'));
$io->writeLine("Skip duplicates: " . ($skipDuplicates ? 'YES' : 'NO'));
$io->writeLine("Dry run: " . ($dryRun ? 'YES' : 'NO'));
$io->writeLine('');
$this->importService->import($input, [
'batchSize' => $batchSize,
'preserveIds' => $preserveIds,
'skipDuplicates' => $skipDuplicates,
'dryRun' => $dryRun,
'verbose' => $verbose,
], $io);
}
}
files/custom/Espo/Modules/DataMigration/Commands/DataMigrationStatus.php
+167
View File
@@ -0,0 +1,167 @@
<?php
namespace Espo\Modules\DataMigration\Commands;
use Espo\Core\Console\Command;
use Espo\Core\Console\Command\Params;
use Espo\Core\Console\IO;
use Espo\ORM\EntityManager;
use Espo\Modules\DataMigration\Services\ManifestBuilder;
class DataMigrationStatus implements Command
{
private const ENTITY_TYPES = [
'Case',
'Account',
'Contact',
'CaseActivity',
'Charge',
'Invoice',
'Document',
'NhMeeting',
'NhDecision',
'NhProcess',
'NhPlea',
'NhActivity',
'NhDocument',
'NhSyncLog',
'CaseMemory',
'SignatureRequest',
'SignatureRequestSigner',
'SmsLog',
'PricingAgreement',
'PricingAgreementRate',
'FixedActivityRate',
'ClientRate',
];
public function __construct(
private EntityManager $entityManager,
private ManifestBuilder $manifestBuilder
) {}
public function run(Params $params, IO $io): void
{
$inputDir = $params->getOption('input');
if ($inputDir) {
$this->showBackupStatus($inputDir, $io);
} else {
$this->showLiveStatus($io);
}
}
private function showLiveStatus(IO $io): void
{
$io->writeLine('EspoCRM Data Migration - Live Database Status');
$io->writeLine('=============================================');
$io->writeLine('');
$total = 0;
foreach (self::ENTITY_TYPES as $entityType) {
try {
$count = $this->entityManager
->getRDBRepository($entityType)
->count();
if ($count > 0) {
$io->writeLine(sprintf(" %-30s %d", $entityType, $count));
$total += $count;
}
} catch (\Exception $e) {
// Entity type doesn't exist (module not installed)
}
}
$io->writeLine('');
$io->writeLine("Total records: {$total}");
// Count attachments on disk
$uploadDir = 'data/upload/';
if (is_dir($uploadDir)) {
$fileCount = 0;
$totalSize = 0;
$files = scandir($uploadDir);
foreach ($files as $file) {
if ($file === '.' || $file === '..') {
continue;
}
$path = $uploadDir . $file;
if (is_file($path)) {
$fileCount++;
$totalSize += filesize($path);
}
}
$io->writeLine('');
$io->writeLine("Attachment files: {$fileCount} (" . $this->formatBytes($totalSize) . ")");
}
}
private function showBackupStatus(string $inputDir, IO $io): void
{
$inputDir = rtrim($inputDir, '/');
$io->writeLine('EspoCRM Data Migration - Backup Status');
$io->writeLine('======================================');
$io->writeLine("Path: {$inputDir}");
$io->writeLine('');
$manifest = $this->manifestBuilder->read($inputDir);
if ($manifest === null) {
$io->writeLine('ERROR: No manifest.json found.');
return;
}
$io->writeLine("Source: " . ($manifest['sourceHost'] ?? 'unknown'));
$io->writeLine("Exported at: " . ($manifest['exportedAt'] ?? 'unknown'));
$io->writeLine("EspoCRM version: " . ($manifest['espoVersion'] ?? 'unknown'));
$io->writeLine('');
if (!empty($manifest['modules'])) {
$io->writeLine('Modules:');
foreach ($manifest['modules'] as $module => $version) {
$io->writeLine(" {$module}: {$version}");
}
$io->writeLine('');
}
$io->writeLine('Entity counts:');
foreach ($manifest['entityCounts'] ?? [] as $type => $count) {
if ($count > 0) {
$io->writeLine(sprintf(" %-30s %d", $type, $count));
}
}
$io->writeLine('');
$io->writeLine("Attachment files: " . ($manifest['totalAttachmentFiles'] ?? 0));
$io->writeLine("Attachment size: " . $this->formatBytes($manifest['totalAttachmentSizeBytes'] ?? 0));
}
private function formatBytes(int $bytes): string
{
if ($bytes >= 1073741824) {
return round($bytes / 1073741824, 2) . ' GB';
}
if ($bytes >= 1048576) {
return round($bytes / 1048576, 2) . ' MB';
}
if ($bytes >= 1024) {
return round($bytes / 1024, 2) . ' KB';
}
return $bytes . ' B';
}
}
files/custom/Espo/Modules/DataMigration/Resources/metadata/app/consoleCommands.json
+20
View File
@@ -0,0 +1,20 @@
{
"dataMigrationExport": {
"className": "Espo\\Modules\\DataMigration\\Commands\\DataMigrationExport",
"listed": true,
"allowedOptions": ["output", "batchSize", "caseIds", "status"],
"allowedFlags": ["dryRun", "noFiles", "verbose"]
},
"dataMigrationImport": {
"className": "Espo\\Modules\\DataMigration\\Commands\\DataMigrationImport",
"listed": true,
"allowedOptions": ["input", "batchSize"],
"allowedFlags": ["dryRun", "preserveIds", "skipDuplicates", "verbose"]
},
"dataMigrationStatus": {
"className": "Espo\\Modules\\DataMigration\\Commands\\DataMigrationStatus",
"listed": true,
"allowedOptions": ["input"],
"allowedFlags": []
}
}
files/custom/Espo/Modules/DataMigration/Resources/module.json
+8
View File
@@ -0,0 +1,8 @@
{
"order": 99,
"version": "1.0.0",
"requires": {
"espocrm": ">=8.0.0"
},
"description": "Data migration tool for case data backup and restore"
}
files/custom/Espo/Modules/DataMigration/Services/AttachmentHandler.php
+150
View File
@@ -0,0 +1,150 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\Core\Console\IO;
class AttachmentHandler
{
private const UPLOAD_DIR = 'data/upload/';
/**
* Export attachment files to the backup directory.
*
* @param string[] $attachmentIds
* @param string $outputDir Backup files/ directory
* @param bool $verbose
* @param IO|null $io
* @return array{count: int, size: int} Number of files copied and total bytes
*/
public function exportFiles(
array $attachmentIds,
string $outputDir,
bool $verbose = false,
?IO $io = null
): array {
$filesDir = rtrim($outputDir, '/');
if (!is_dir($filesDir)) {
mkdir($filesDir, 0755, true);
}
$count = 0;
$totalSize = 0;
$missing = 0;
foreach ($attachmentIds as $id) {
$sourcePath = self::UPLOAD_DIR . $id;
if (!file_exists($sourcePath)) {
$missing++;
if ($verbose && $io) {
$io->writeLine(" WARNING: Attachment file not found: {$id}");
}
continue;
}
$destPath = $filesDir . '/' . $id;
copy($sourcePath, $destPath);
$size = filesize($sourcePath);
$totalSize += $size;
$count++;
}
if ($io) {
$io->writeLine(" Attachments: copied {$count} files (" . $this->formatBytes($totalSize) . ")");
if ($missing > 0) {
$io->writeLine(" WARNING: {$missing} attachment files not found on disk");
}
}
return ['count' => $count, 'size' => $totalSize];
}
/**
* Import attachment files from backup to data/upload/.
*
* @param string $inputDir Backup files/ directory
* @param IdMapper $idMapper
* @param bool $preserveIds
* @param bool $verbose
* @param IO|null $io
* @return int Number of files restored
*/
public function importFiles(
string $inputDir,
IdMapper $idMapper,
bool $preserveIds = true,
bool $verbose = false,
?IO $io = null
): int {
$filesDir = rtrim($inputDir, '/');
if (!is_dir($filesDir)) {
if ($io) {
$io->writeLine(" No files/ directory in backup, skipping attachment restore.");
}
return 0;
}
if (!is_dir(self::UPLOAD_DIR)) {
mkdir(self::UPLOAD_DIR, 0755, true);
}
$files = scandir($filesDir);
$count = 0;
foreach ($files as $file) {
if ($file === '.' || $file === '..') {
continue;
}
$sourcePath = $filesDir . '/' . $file;
if (!is_file($sourcePath)) {
continue;
}
if ($preserveIds) {
$destId = $file;
} else {
$destId = $idMapper->get('Attachment', $file) ?? $file;
}
$destPath = self::UPLOAD_DIR . $destId;
if (!file_exists($destPath)) {
copy($sourcePath, $destPath);
$count++;
}
}
if ($io) {
$io->writeLine(" Attachments: restored {$count} files");
}
return $count;
}
private function formatBytes(int $bytes): string
{
if ($bytes >= 1073741824) {
return round($bytes / 1073741824, 2) . ' GB';
}
if ($bytes >= 1048576) {
return round($bytes / 1048576, 2) . ' MB';
}
if ($bytes >= 1024) {
return round($bytes / 1024, 2) . ' KB';
}
return $bytes . ' B';
}
}
files/custom/Espo/Modules/DataMigration/Services/EntityExporter.php
+93
View File
@@ -0,0 +1,93 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\ORM\EntityManager;
use Espo\Core\Console\IO;
class EntityExporter
{
public function __construct(
private EntityManager $entityManager
) {}
/**
* Export entities of a given type matching the where clause, writing to a JSON file.
*
* @param string $entityType
* @param array $where ORM where conditions
* @param string $outputDir Backup data/ directory
* @param int $batchSize
* @param bool $verbose
* @param IO|null $io
* @return int Number of records exported
*/
public function export(
string $entityType,
array $where,
string $outputDir,
int $batchSize = 500,
bool $verbose = false,
?IO $io = null
): int {
$filePath = rtrim($outputDir, '/') . '/' . $entityType . '.json';
$records = [];
$offset = 0;
$total = 0;
while (true) {
$query = $this->entityManager
->getRDBRepository($entityType)
->where($where)
->limit($offset, $batchSize);
$collection = $query->find();
$count = 0;
foreach ($collection as $entity) {
$record = $this->entityToArray($entity);
$records[] = $record;
$count++;
$total++;
}
if ($count === 0) {
break;
}
if ($verbose && $io) {
$io->writeLine(" {$entityType}: exported {$total} records...");
}
$offset += $batchSize;
if ($count < $batchSize) {
break;
}
}
$data = [
'entityType' => $entityType,
'count' => $total,
'exportedAt' => date('c'),
'records' => $records,
];
file_put_contents(
$filePath,
json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
);
return $total;
}
/**
* Convert an entity to a plain associative array using getValueMap().
*/
private function entityToArray($entity): array
{
$valueMap = $entity->getValueMap();
return (array) $valueMap;
}
}
files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php
+384
View File
@@ -0,0 +1,384 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\ORM\EntityManager;
use Espo\Core\Console\IO;
use Espo\Core\Utils\Metadata;
use PDO;
class EntityImporter
{
/**
* Fields to remap as foreign keys per entity type.
* Format: entityType => [field => referencedEntityType]
*/
private const FK_FIELDS = [
'Case' => [
'accountId' => 'Account',
'contactId' => 'Contact',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
'cPayerId' => 'Account',
'cBeneficiaryId' => 'Contact',
'cPricingAgreementId' => 'PricingAgreement',
],
'CaseActivity' => [
'caseId' => 'Case',
'accountId' => 'Account',
'invoiceId' => 'Invoice',
'chargeId' => 'Charge',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
'Charge' => [
'caseId' => 'Case',
'accountId' => 'Account',
'caseActivityId' => 'CaseActivity',
'invoiceId' => 'Invoice',
'fixedActivityRateId' => 'FixedActivityRate',
'pricingAgreementRateId' => 'PricingAgreementRate',
'legalAidPayerAccountId' => 'Account',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
'Invoice' => [
'caseId' => 'Case',
'accountId' => 'Account',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
'Document' => [
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
'Note' => [
'parentId' => '_parent', // special: uses parentType
'createdById' => 'User',
'modifiedById' => 'User',
],
'Attachment' => [
'parentId' => '_parent',
'createdById' => 'User',
],
'NhMeeting' => ['caseId' => 'Case'],
'NhDecision' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'],
'NhProcess' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'],
'NhPlea' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'],
'NhActivity' => ['caseId' => 'Case', 'taskId' => 'Task'],
'NhDocument' => ['caseId' => 'Case'],
'NhSyncLog' => ['caseId' => 'Case'],
'CaseMemory' => [
'caseId' => 'Case',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
'SignatureRequest' => [
'caseId' => 'Case',
'documentId' => 'Document',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
'signedDocumentId' => 'Attachment',
],
'SignatureRequestSigner' => [
'signatureRequestId' => 'SignatureRequest',
'contactId' => 'Contact',
'userId' => 'User',
],
'SmsLog' => [
'parentId' => '_parent',
'createdById' => 'User',
],
'PricingAgreement' => [
'accountId' => 'Account',
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
'PricingAgreementRate' => [
'pricingAgreementId' => 'PricingAgreement',
],
'ClientRate' => [
'accountId' => 'Account',
],
'Contact' => [
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
'accountId' => 'Account',
],
'Account' => [
'assignedUserId' => 'User',
'createdById' => 'User',
'modifiedById' => 'User',
],
];
/**
* Fields to skip during import (non-storable, computed, etc.).
*/
private const SKIP_FIELDS = [
'deleted',
'totalBillableAmount',
'unbilledAmount',
'contactRole',
'contactPartyOrder',
'caseRole',
'casePartyOrder',
'cLastActivityIndicator',
'signerCount',
'signedCount',
];
/**
* Entity types that have an autoincrement 'number' field.
*/
private const AUTOINCREMENT_ENTITIES = ['Case', 'Charge', 'Invoice'];
public function __construct(
private EntityManager $entityManager,
private Metadata $metadata
) {}
/**
* Import a single entity type from its JSON file.
*
* @return array{imported: int, skipped: int, errors: int}
*/
public function import(
string $entityType,
string $jsonFilePath,
bool $preserveIds,
bool $skipDuplicates,
IdMapper $idMapper,
int $batchSize,
bool $verbose = false,
bool $dryRun = false,
?IO $io = null
): array {
if (!file_exists($jsonFilePath)) {
return ['imported' => 0, 'skipped' => 0, 'errors' => 0];
}
$content = json_decode(file_get_contents($jsonFilePath), true);
if (!$content || empty($content['records'])) {
return ['imported' => 0, 'skipped' => 0, 'errors' => 0];
}
$records = $content['records'];
$imported = 0;
$skipped = 0;
$errors = 0;
$hasAutoIncrement = in_array($entityType, self::AUTOINCREMENT_ENTITIES);
foreach ($records as $record) {
$oldId = $record['id'] ?? null;
if (!$oldId) {
$errors++;
continue;
}
// Check for duplicates
if ($preserveIds) {
$existing = $this->entityManager
->getRDBRepository($entityType)
->where(['id' => $oldId])
->findOne();
if ($existing) {
if ($skipDuplicates) {
$skipped++;
$idMapper->add($entityType, $oldId, $oldId);
continue;
}
$errors++;
if ($verbose && $io) {
$io->writeLine(" ERROR: {$entityType} ID {$oldId} already exists");
}
continue;
}
}
if ($dryRun) {
$newId = $preserveIds ? $oldId : $idMapper->generateId();
$idMapper->add($entityType, $oldId, $newId);
$imported++;
continue;
}
try {
$newId = $this->importRecord($entityType, $record, $preserveIds, $idMapper, $hasAutoIncrement);
$idMapper->add($entityType, $oldId, $newId);
$imported++;
} catch (\Exception $e) {
$errors++;
if ($verbose && $io) {
$io->writeLine(" ERROR importing {$entityType} {$oldId}: " . $e->getMessage());
}
}
}
// Reset autoincrement counter
if (!$dryRun && $hasAutoIncrement && $imported > 0) {
$this->resetAutoIncrement($entityType);
}
return ['imported' => $imported, 'skipped' => $skipped, 'errors' => $errors];
}
private function importRecord(
string $entityType,
array $record,
bool $preserveIds,
IdMapper $idMapper,
bool $hasAutoIncrement
): string {
$oldId = $record['id'];
$newId = $preserveIds ? $oldId : $idMapper->generateId();
// Remap foreign keys if not preserving IDs
if (!$preserveIds) {
$fkMap = self::FK_FIELDS[$entityType] ?? [];
foreach ($fkMap as $field => $refType) {
if (!isset($record[$field]) || $record[$field] === null) {
continue;
}
if ($refType === '_parent') {
// Polymorphic: use parentType to determine entity type
$parentTypeField = str_replace('Id', 'Type', $field);
$parentType = $record[$parentTypeField] ?? null;
if ($parentType) {
$mapped = $idMapper->get($parentType, $record[$field]);
if ($mapped) {
$record[$field] = $mapped;
}
}
} else {
$mapped = $idMapper->get($refType, $record[$field]);
if ($mapped) {
$record[$field] = $mapped;
}
}
}
}
// For entities with autoincrement number, use raw SQL
if ($hasAutoIncrement) {
$this->insertWithRawSql($entityType, $record, $newId);
return $newId;
}
// Use ORM for standard entities
$entity = $this->entityManager->getRDBRepository($entityType)->getNew();
$entity->set('id', $newId);
foreach ($record as $field => $value) {
if ($field === 'id' || in_array($field, self::SKIP_FIELDS)) {
continue;
}
$entity->set($field, $value);
}
$this->entityManager->saveEntity($entity, [
'silent' => true,
'skipHooks' => true,
'noStream' => true,
'skipAll' => true,
]);
return $newId;
}
private function insertWithRawSql(string $entityType, array $record, string $newId): void
{
$tableName = $this->entityManager
->getQueryComposer()
->toDb($entityType);
// Convert camelCase fields to snake_case for SQL
$columns = [];
$values = [];
$placeholders = [];
$record['id'] = $newId;
foreach ($record as $field => $value) {
if (in_array($field, self::SKIP_FIELDS)) {
continue;
}
// Skip non-storable array/object fields that come from getValueMap
if (is_array($value) || is_object($value)) {
// JSON fields should be stored as JSON strings
if (is_array($value) || is_object($value)) {
$value = json_encode($value, JSON_UNESCAPED_UNICODE);
}
}
$column = $this->camelCaseToUnderscore($field);
$columns[] = "`{$column}`";
$values[] = $value;
$placeholders[] = '?';
}
$columnStr = implode(', ', $columns);
$placeholderStr = implode(', ', $placeholders);
$sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
try {
$pdo = $this->entityManager->getPDO();
$stmt = $pdo->prepare($sql);
$stmt->execute($values);
} catch (\Exception $e) {
// If column mismatch, try with just known columns
throw $e;
}
}
private function resetAutoIncrement(string $entityType): void
{
$tableName = $this->entityManager
->getQueryComposer()
->toDb($entityType);
try {
$pdo = $this->entityManager->getPDO();
$sql = "SELECT MAX(`number`) as max_num FROM `{$tableName}`";
$sth = $pdo->query($sql);
$row = $sth->fetch(\PDO::FETCH_ASSOC);
if ($row && $row['max_num']) {
$next = (int) $row['max_num'] + 1;
$pdo->exec("ALTER TABLE `{$tableName}` AUTO_INCREMENT = {$next}");
}
} catch (\Exception $e) {
// Non-critical, log and continue
}
}
private function camelCaseToUnderscore(string $str): string
{
return strtolower(preg_replace('/(?<!^)[A-Z]/', '_$0', $str));
}
}
files/custom/Espo/Modules/DataMigration/Services/ExportService.php
+510
View File
@@ -0,0 +1,510 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\ORM\EntityManager;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\Log;
use Espo\Core\Console\IO;
use PDO;
class ExportService
{
public function __construct(
private EntityManager $entityManager,
private EntityExporter $entityExporter,
private JunctionTableHandler $junctionTableHandler,
private AttachmentHandler $attachmentHandler,
private ManifestBuilder $manifestBuilder,
private Config $config,
private Log $log
) {}
public function export(string $outputDir, array $options, IO $io): void
{
$batchSize = $options['batchSize'] ?? 500;
$filterCaseIds = $options['caseIds'] ?? null;
$filterStatus = $options['caseStatus'] ?? null;
$noFiles = $options['noFiles'] ?? false;
$verbose = $options['verbose'] ?? false;
$dataDir = $outputDir . '/data';
$filesDir = $outputDir . '/files';
mkdir($dataDir, 0755, true);
if (!$noFiles) {
mkdir($filesDir, 0755, true);
}
$entityCounts = [];
$entityIdsByType = [];
$io->writeLine('Starting export...');
$io->writeLine('');
// === Determine Case IDs ===
$caseWhere = [];
if ($filterCaseIds) {
$caseWhere['id'] = $filterCaseIds;
}
if ($filterStatus) {
$caseWhere['status'] = $filterStatus;
}
$caseIds = $this->collectIds('Case', $caseWhere);
$entityIdsByType['Case'] = $caseIds;
$io->writeLine("Found " . count($caseIds) . " cases to export.");
if (empty($caseIds)) {
$io->writeLine('No cases found. Nothing to export.');
return;
}
// === Layer 0: Users, Teams, FixedActivityRate ===
$io->writeLine('');
$io->writeLine('[Layer 0] Reference entities...');
$referencedUserIds = $this->collectReferencedUserIds($caseIds);
$entityIdsByType['User'] = $referencedUserIds;
if (!empty($referencedUserIds)) {
$entityCounts['User'] = $this->entityExporter->export(
'User', ['id' => $referencedUserIds], $dataDir, $batchSize, $verbose, $io
);
}
$referencedTeamIds = $this->collectReferencedTeamIds($caseIds);
$entityIdsByType['Team'] = $referencedTeamIds;
if (!empty($referencedTeamIds)) {
$entityCounts['Team'] = $this->entityExporter->export(
'Team', ['id' => $referencedTeamIds], $dataDir, $batchSize, $verbose, $io
);
}
if ($this->entityTypeExists('FixedActivityRate')) {
$entityCounts['FixedActivityRate'] = $this->entityExporter->export(
'FixedActivityRate', [], $dataDir, $batchSize, $verbose, $io
);
}
// === Layer 1: Accounts and Contacts ===
$io->writeLine('');
$io->writeLine('[Layer 1] Accounts and Contacts...');
$accountIds = $this->collectReferencedAccountIds($caseIds);
$entityIdsByType['Account'] = $accountIds;
if (!empty($accountIds)) {
$entityCounts['Account'] = $this->entityExporter->export(
'Account', ['id' => $accountIds], $dataDir, $batchSize, $verbose, $io
);
}
$contactIds = $this->collectReferencedContactIds($caseIds);
$entityIdsByType['Contact'] = $contactIds;
if (!empty($contactIds)) {
$entityCounts['Contact'] = $this->entityExporter->export(
'Contact', ['id' => $contactIds], $dataDir, $batchSize, $verbose, $io
);
}
// === Layer 2: Billing rates ===
$io->writeLine('');
$io->writeLine('[Layer 2] Billing rates...');
if ($this->entityTypeExists('PricingAgreement') && !empty($accountIds)) {
$entityCounts['PricingAgreement'] = $this->entityExporter->export(
'PricingAgreement', ['accountId' => $accountIds], $dataDir, $batchSize, $verbose, $io
);
$paIds = $this->collectIds('PricingAgreement', ['accountId' => $accountIds]);
$entityIdsByType['PricingAgreement'] = $paIds;
if (!empty($paIds) && $this->entityTypeExists('PricingAgreementRate')) {
$entityCounts['PricingAgreementRate'] = $this->entityExporter->export(
'PricingAgreementRate', ['pricingAgreementId' => $paIds], $dataDir, $batchSize, $verbose, $io
);
}
}
if ($this->entityTypeExists('ClientRate') && !empty($accountIds)) {
$entityCounts['ClientRate'] = $this->entityExporter->export(
'ClientRate', ['accountId' => $accountIds], $dataDir, $batchSize, $verbose, $io
);
}
// === Layer 3: Cases ===
$io->writeLine('');
$io->writeLine('[Layer 3] Cases...');
$entityCounts['Case'] = $this->entityExporter->export(
'Case', ['id' => $caseIds], $dataDir, $batchSize, $verbose, $io
);
// === Layer 4: Direct Case children ===
$io->writeLine('');
$io->writeLine('[Layer 4] Case children...');
$layer4Entities = [
'CaseActivity', 'Invoice', 'NhMeeting', 'NhDecision',
'NhProcess', 'NhPlea', 'NhActivity', 'NhDocument',
'NhSyncLog', 'CaseMemory', 'SignatureRequest',
];
foreach ($layer4Entities as $entityType) {
if (!$this->entityTypeExists($entityType)) {
continue;
}
$entityCounts[$entityType] = $this->entityExporter->export(
$entityType, ['caseId' => $caseIds], $dataDir, $batchSize, $verbose, $io
);
$entityIdsByType[$entityType] = $this->collectIds($entityType, ['caseId' => $caseIds]);
}
// Document - linked via junction table
if ($this->entityTypeExists('Document')) {
$documentIds = $this->collectDocumentIdsForCases($caseIds);
$entityIdsByType['Document'] = $documentIds;
if (!empty($documentIds)) {
$entityCounts['Document'] = $this->entityExporter->export(
'Document', ['id' => $documentIds], $dataDir, $batchSize, $verbose, $io
);
}
}
// === Layer 5: Second-level children ===
$io->writeLine('');
$io->writeLine('[Layer 5] Second-level children...');
if ($this->entityTypeExists('Charge')) {
$entityCounts['Charge'] = $this->entityExporter->export(
'Charge', ['caseId' => $caseIds], $dataDir, $batchSize, $verbose, $io
);
$entityIdsByType['Charge'] = $this->collectIds('Charge', ['caseId' => $caseIds]);
}
if ($this->entityTypeExists('SignatureRequestSigner')) {
$srIds = $entityIdsByType['SignatureRequest'] ?? [];
if (!empty($srIds)) {
$entityCounts['SignatureRequestSigner'] = $this->entityExporter->export(
'SignatureRequestSigner', ['signatureRequestId' => $srIds], $dataDir, $batchSize, $verbose, $io
);
}
}
if ($this->entityTypeExists('SmsLog')) {
$entityCounts['SmsLog'] = $this->entityExporter->export(
'SmsLog', ['parentType' => 'Case', 'parentId' => $caseIds], $dataDir, $batchSize, $verbose, $io
);
}
// === Layer 6: Notes and Attachments ===
$io->writeLine('');
$io->writeLine('[Layer 6] Notes and Attachments...');
$entityCounts['Note'] = $this->entityExporter->export(
'Note', ['parentType' => 'Case', 'parentId' => $caseIds], $dataDir, $batchSize, $verbose, $io
);
$attachmentIds = $this->collectAllAttachmentIds($dataDir);
$entityIdsByType['Attachment'] = $attachmentIds;
if (!empty($attachmentIds)) {
$entityCounts['Attachment'] = $this->entityExporter->export(
'Attachment', ['id' => $attachmentIds], $dataDir, $batchSize, $verbose, $io
);
}
// === Junction tables ===
$io->writeLine('');
$io->writeLine('[Junctions] Relationship tables...');
$this->junctionTableHandler->exportJunctions($caseIds, $entityIdsByType, $dataDir, $io);
// === Files ===
$fileCount = 0;
$fileSize = 0;
if (!$noFiles && !empty($attachmentIds)) {
$io->writeLine('');
$io->writeLine('[Files] Copying attachment files...');
$result = $this->attachmentHandler->exportFiles($attachmentIds, $filesDir, $verbose, $io);
$fileCount = $result['count'];
$fileSize = $result['size'];
}
// === Manifest ===
$this->manifestBuilder->build($outputDir, $entityCounts, $fileCount, $fileSize);
$io->writeLine('');
$io->writeLine('=== Export Complete ===');
$io->writeLine("Output: {$outputDir}");
foreach ($entityCounts as $type => $count) {
if ($count > 0) {
$io->writeLine(" {$type}: {$count}");
}
}
}
private function collectIds(string $entityType, array $where): array
{
$ids = [];
$collection = $this->entityManager
->getRDBRepository($entityType)
->select(['id'])
->where($where)
->find();
foreach ($collection as $entity) {
$ids[] = $entity->getId();
}
return $ids;
}
private function collectReferencedUserIds(array $caseIds): array
{
if (empty($caseIds)) {
return [];
}
$userIds = [];
$pdo = $this->entityManager->getPDO();
$inList = $this->buildInList($caseIds);
// From Cases - all user reference fields
$sql = "SELECT DISTINCT assigned_user_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0
UNION
SELECT DISTINCT created_by_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0
UNION
SELECT DISTINCT modified_by_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0";
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
if ($row[0]) {
$userIds[$row[0]] = true;
}
}
// From child entities
$childTables = ['case_activity', 'charge', 'invoice'];
foreach ($childTables as $table) {
if (!$this->tableExists($table)) {
continue;
}
$sql = "SELECT DISTINCT assigned_user_id FROM `{$table}` WHERE case_id IN ({$inList}) AND deleted = 0
UNION
SELECT DISTINCT created_by_id FROM `{$table}` WHERE case_id IN ({$inList}) AND deleted = 0";
try {
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
if ($row[0]) {
$userIds[$row[0]] = true;
}
}
} catch (\Exception $e) {
// Table may not have these columns
}
}
return array_keys($userIds);
}
private function collectReferencedTeamIds(array $caseIds): array
{
if (empty($caseIds)) {
return [];
}
$pdo = $this->entityManager->getPDO();
$inList = $this->buildInList($caseIds);
$sql = "SELECT DISTINCT team_id FROM `entity_team` WHERE entity_type = 'Case' AND entity_id IN ({$inList}) AND deleted = 0";
$sth = $pdo->query($sql);
$teamIds = [];
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
if ($row[0]) {
$teamIds[] = $row[0];
}
}
return $teamIds;
}
private function collectReferencedAccountIds(array $caseIds): array
{
if (empty($caseIds)) {
return [];
}
$pdo = $this->entityManager->getPDO();
$inList = $this->buildInList($caseIds);
$accountIds = [];
$sql = "SELECT DISTINCT account_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 AND account_id IS NOT NULL";
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
$accountIds[$row[0]] = true;
}
// cPayerId from GreenInvoiceBilling
try {
$sql = "SELECT DISTINCT c_payer_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 AND c_payer_id IS NOT NULL";
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
$accountIds[$row[0]] = true;
}
} catch (\Exception $e) {
// Column may not exist
}
return array_keys($accountIds);
}
private function collectReferencedContactIds(array $caseIds): array
{
if (empty($caseIds)) {
return [];
}
$pdo = $this->entityManager->getPDO();
$inList = $this->buildInList($caseIds);
$contactIds = [];
$sql = "SELECT DISTINCT contact_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 AND contact_id IS NOT NULL";
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
$contactIds[$row[0]] = true;
}
$sql = "SELECT DISTINCT contact_id FROM `case_contact` WHERE case_id IN ({$inList}) AND deleted = 0";
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
$contactIds[$row[0]] = true;
}
return array_keys($contactIds);
}
private function collectDocumentIdsForCases(array $caseIds): array
{
if (empty($caseIds)) {
return [];
}
$pdo = $this->entityManager->getPDO();
$inList = $this->buildInList($caseIds);
$sql = "SELECT DISTINCT document_id FROM `case_document` WHERE case_id IN ({$inList}) AND deleted = 0";
$sth = $pdo->query($sql);
$ids = [];
while ($row = $sth->fetch(PDO::FETCH_NUM)) {
$ids[] = $row[0];
}
return $ids;
}
private function collectAllAttachmentIds(string $dataDir): array
{
$attachmentIds = [];
$files = glob($dataDir . '/*.json');
foreach ($files as $file) {
$basename = basename($file);
if (str_starts_with($basename, '_junction_')) {
continue;
}
$content = json_decode(file_get_contents($file), true);
if (!$content || empty($content['records'])) {
continue;
}
foreach ($content['records'] as $record) {
// Known file fields
$knownFileFields = ['fileId', 'signedDocumentId'];
foreach ($knownFileFields as $ff) {
if (!empty($record[$ff])) {
$attachmentIds[$record[$ff]] = true;
}
}
// attachmentsIds (attachmentMultiple)
if (!empty($record['attachmentsIds']) && is_array($record['attachmentsIds'])) {
foreach ($record['attachmentsIds'] as $attId) {
$attachmentIds[$attId] = true;
}
}
// Note data field
if (isset($record['data'])) {
$data = is_array($record['data']) ? $record['data'] : (array) $record['data'];
if (!empty($data['attachmentsIds']) && is_array($data['attachmentsIds'])) {
foreach ($data['attachmentsIds'] as $attId) {
$attachmentIds[$attId] = true;
}
}
}
}
}
return array_keys($attachmentIds);
}
/**
* Build a quoted IN list from an array of IDs.
*/
private function buildInList(array $ids): string
{
$pdo = $this->entityManager->getPDO();
return implode(',', array_map(fn($id) => $pdo->quote($id), $ids));
}
private function entityTypeExists(string $entityType): bool
{
try {
$this->entityManager->getRDBRepository($entityType);
return true;
} catch (\Exception $e) {
return false;
}
}
private function tableExists(string $table): bool
{
try {
$this->entityManager->getSqlExecutor()->execute("SELECT 1 FROM `{$table}` LIMIT 1");
return true;
} catch (\Exception $e) {
return false;
}
}
}
files/custom/Espo/Modules/DataMigration/Services/IdMapper.php
+80
View File
@@ -0,0 +1,80 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\Core\Utils\Util;
class IdMapper
{
/** @var array<string, array<string, string>> [entityType][oldId] => newId */
private array $map = [];
public function add(string $entityType, string $oldId, string $newId): void
{
$this->map[$entityType][$oldId] = $newId;
}
public function get(string $entityType, ?string $oldId): ?string
{
if ($oldId === null) {
return null;
}
return $this->map[$entityType][$oldId] ?? null;
}
public function has(string $entityType, string $oldId): bool
{
return isset($this->map[$entityType][$oldId]);
}
/**
* Try to resolve an ID across multiple entity types.
* Useful for polymorphic parent fields.
*/
public function resolve(?string $oldId): ?string
{
if ($oldId === null) {
return null;
}
foreach ($this->map as $entries) {
if (isset($entries[$oldId])) {
return $entries[$oldId];
}
}
return null;
}
public function generateId(): string
{
return Util::generateId();
}
public function getMap(): array
{
return $this->map;
}
public function saveTo(string $path): void
{
file_put_contents(
$path,
json_encode($this->map, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)
);
}
public function loadFrom(string $path): void
{
if (!file_exists($path)) {
return;
}
$data = json_decode(file_get_contents($path), true);
if (is_array($data)) {
$this->map = $data;
}
}
}
files/custom/Espo/Modules/DataMigration/Services/ImportService.php
+174
View File
@@ -0,0 +1,174 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\ORM\EntityManager;
use Espo\Core\Utils\Log;
use Espo\Core\Console\IO;
class ImportService
{
/**
* Import order: entities must be imported in dependency order.
*/
private const IMPORT_ORDER = [
'User',
'Team',
'FixedActivityRate',
'Account',
'Contact',
'PricingAgreement',
'PricingAgreementRate',
'ClientRate',
'Case',
'CaseActivity',
'Invoice',
'Document',
'NhDocument',
'NhMeeting',
'NhDecision',
'NhProcess',
'NhPlea',
'NhActivity',
'NhSyncLog',
'CaseMemory',
'SignatureRequest',
'Charge',
'SignatureRequestSigner',
'SmsLog',
'Note',
'Attachment',
];
public function __construct(
private EntityManager $entityManager,
private EntityImporter $entityImporter,
private JunctionTableHandler $junctionTableHandler,
private AttachmentHandler $attachmentHandler,
private ManifestBuilder $manifestBuilder,
private Log $log
) {}
/**
* Run full import from backup directory.
*
* @param string $inputDir Backup directory path
* @param array $options {batchSize: int, preserveIds: bool, skipDuplicates: bool, dryRun: bool, verbose: bool}
* @param IO $io
*/
public function import(string $inputDir, array $options, IO $io): void
{
$batchSize = $options['batchSize'] ?? 500;
$preserveIds = $options['preserveIds'] ?? true;
$skipDuplicates = $options['skipDuplicates'] ?? false;
$dryRun = $options['dryRun'] ?? false;
$verbose = $options['verbose'] ?? false;
$dataDir = rtrim($inputDir, '/') . '/data';
$filesDir = rtrim($inputDir, '/') . '/files';
// Read and display manifest
$manifest = $this->manifestBuilder->read($inputDir);
if ($manifest === null) {
$io->writeLine('ERROR: No manifest.json found in backup directory.');
return;
}
$io->writeLine('=== Backup Manifest ===');
$io->writeLine("Source: " . ($manifest['sourceHost'] ?? 'unknown'));
$io->writeLine("Exported at: " . ($manifest['exportedAt'] ?? 'unknown'));
$io->writeLine("EspoCRM version: " . ($manifest['espoVersion'] ?? 'unknown'));
$io->writeLine('');
if ($dryRun) {
$io->writeLine('*** DRY RUN MODE - No changes will be made ***');
$io->writeLine('');
}
$idMapper = new IdMapper();
$totalImported = 0;
$totalSkipped = 0;
$totalErrors = 0;
$results = [];
// Import entities in order
foreach (self::IMPORT_ORDER as $entityType) {
$jsonFile = $dataDir . '/' . $entityType . '.json';
if (!file_exists($jsonFile)) {
continue;
}
$io->writeLine("Importing {$entityType}...");
$result = $this->entityImporter->import(
$entityType,
$jsonFile,
$preserveIds,
$skipDuplicates,
$idMapper,
$batchSize,
$verbose,
$dryRun,
$io
);
$results[$entityType] = $result;
$totalImported += $result['imported'];
$totalSkipped += $result['skipped'];
$totalErrors += $result['errors'];
$io->writeLine(" {$entityType}: imported={$result['imported']}, skipped={$result['skipped']}, errors={$result['errors']}");
}
// Import junction tables
$io->writeLine('');
$io->writeLine('Importing junction tables...');
if (!$dryRun) {
$this->junctionTableHandler->importJunctions($dataDir, $idMapper, $preserveIds, $io);
} else {
$io->writeLine(' (skipped in dry-run mode)');
}
// Restore attachment files
if (is_dir($filesDir)) {
$io->writeLine('');
$io->writeLine('Restoring attachment files...');
if (!$dryRun) {
$this->attachmentHandler->importFiles($filesDir, $idMapper, $preserveIds, $verbose, $io);
} else {
$fileCount = count(array_diff(scandir($filesDir), ['.', '..']));
$io->writeLine(" Would restore {$fileCount} files (dry-run)");
}
}
// Save ID mapping for reference
if (!$dryRun && !$preserveIds) {
$mapPath = rtrim($inputDir, '/') . '/id-mapping.json';
$idMapper->saveTo($mapPath);
$io->writeLine('');
$io->writeLine("ID mapping saved to: {$mapPath}");
}
// Summary
$io->writeLine('');
$io->writeLine('=== Import ' . ($dryRun ? 'Dry Run ' : '') . 'Complete ===');
$io->writeLine("Total imported: {$totalImported}");
$io->writeLine("Total skipped: {$totalSkipped}");
$io->writeLine("Total errors: {$totalErrors}");
if ($totalErrors > 0) {
$io->writeLine('');
$io->writeLine('Entities with errors:');
foreach ($results as $type => $r) {
if ($r['errors'] > 0) {
$io->writeLine(" {$type}: {$r['errors']} errors");
}
}
}
}
}
files/custom/Espo/Modules/DataMigration/Services/JunctionTableHandler.php
+231
View File
@@ -0,0 +1,231 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\ORM\EntityManager;
use Espo\Core\Console\IO;
use PDO;
class JunctionTableHandler
{
private const TEAM_ENTITY_TYPES = [
'Case', 'CaseActivity', 'Invoice', 'Charge',
'Document', 'SignatureRequest', 'CaseMemory', 'PricingAgreement',
];
public function __construct(
private EntityManager $entityManager
) {}
/**
* Export junction tables to JSON files.
*/
public function exportJunctions(
array $caseIds,
array $entityIdsByType,
string $outputDir,
?IO $io = null
): array {
$counts = [];
if (empty($caseIds)) {
return $counts;
}
$pdo = $this->entityManager->getPDO();
// case_contact
$inList = $this->buildInList($caseIds);
$counts['case_contact'] = $this->exportTableQuery(
$pdo,
'case_contact',
"SELECT * FROM `case_contact` WHERE `case_id` IN ({$inList}) AND `deleted` = 0",
$outputDir
);
// case_document
$counts['case_document'] = $this->exportTableQuery(
$pdo,
'case_document',
"SELECT * FROM `case_document` WHERE `case_id` IN ({$inList}) AND `deleted` = 0",
$outputDir
);
// entity_team - for all tracked entity types
$allTeamRecords = [];
foreach (self::TEAM_ENTITY_TYPES as $entityType) {
$ids = $entityIdsByType[$entityType] ?? [];
if (empty($ids)) {
continue;
}
$entityInList = $this->buildInList($ids);
$quotedType = $pdo->quote($entityType);
$sql = "SELECT * FROM `entity_team` WHERE `entity_type` = {$quotedType} AND `entity_id` IN ({$entityInList}) AND `deleted` = 0";
$sth = $pdo->query($sql);
while ($row = $sth->fetch(PDO::FETCH_ASSOC)) {
$allTeamRecords[] = $row;
}
}
$filePath = rtrim($outputDir, '/') . '/_junction_entity_team.json';
file_put_contents(
$filePath,
json_encode([
'table' => 'entity_team',
'count' => count($allTeamRecords),
'exportedAt' => date('c'),
'records' => $allTeamRecords,
], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)
);
$counts['entity_team'] = count($allTeamRecords);
if ($io) {
foreach ($counts as $table => $count) {
$io->writeLine(" Junction {$table}: {$count} records");
}
}
return $counts;
}
/**
* Import junction table records from backup.
*/
public function importJunctions(
string $inputDir,
IdMapper $idMapper,
bool $preserveIds = true,
?IO $io = null
): void {
$this->importCaseContact($inputDir, $idMapper, $preserveIds, $io);
$this->importCaseDocument($inputDir, $idMapper, $preserveIds, $io);
$this->importEntityTeam($inputDir, $idMapper, $preserveIds, $io);
}
private function exportTableQuery(PDO $pdo, string $table, string $sql, string $outputDir): int
{
$sth = $pdo->query($sql);
$records = $sth->fetchAll(PDO::FETCH_ASSOC);
$filePath = rtrim($outputDir, '/') . '/_junction_' . $table . '.json';
file_put_contents(
$filePath,
json_encode([
'table' => $table,
'count' => count($records),
'exportedAt' => date('c'),
'records' => $records,
], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)
);
return count($records);
}
private function importCaseContact(string $inputDir, IdMapper $idMapper, bool $preserveIds, ?IO $io): void
{
$data = $this->readJunctionFile($inputDir, 'case_contact');
if ($data === null) {
return;
}
$pdo = $this->entityManager->getPDO();
$count = 0;
$stmt = $pdo->prepare(
"INSERT IGNORE INTO `case_contact` (`case_id`, `contact_id`, `role`, `party_order`, `deleted`) VALUES (?, ?, ?, ?, 0)"
);
foreach ($data['records'] as $row) {
$caseId = $preserveIds ? $row['case_id'] : ($idMapper->get('Case', $row['case_id']) ?? $row['case_id']);
$contactId = $preserveIds ? $row['contact_id'] : ($idMapper->get('Contact', $row['contact_id']) ?? $row['contact_id']);
$stmt->execute([$caseId, $contactId, $row['role'] ?? null, $row['party_order'] ?? null]);
$count++;
}
if ($io) {
$io->writeLine(" Junction case_contact: imported {$count} records");
}
}
private function importCaseDocument(string $inputDir, IdMapper $idMapper, bool $preserveIds, ?IO $io): void
{
$data = $this->readJunctionFile($inputDir, 'case_document');
if ($data === null) {
return;
}
$pdo = $this->entityManager->getPDO();
$count = 0;
$stmt = $pdo->prepare(
"INSERT IGNORE INTO `case_document` (`case_id`, `document_id`, `deleted`) VALUES (?, ?, 0)"
);
foreach ($data['records'] as $row) {
$caseId = $preserveIds ? $row['case_id'] : ($idMapper->get('Case', $row['case_id']) ?? $row['case_id']);
$documentId = $preserveIds ? $row['document_id'] : ($idMapper->get('Document', $row['document_id']) ?? $row['document_id']);
$stmt->execute([$caseId, $documentId]);
$count++;
}
if ($io) {
$io->writeLine(" Junction case_document: imported {$count} records");
}
}
private function importEntityTeam(string $inputDir, IdMapper $idMapper, bool $preserveIds, ?IO $io): void
{
$data = $this->readJunctionFile($inputDir, 'entity_team');
if ($data === null) {
return;
}
$pdo = $this->entityManager->getPDO();
$count = 0;
$stmt = $pdo->prepare(
"INSERT IGNORE INTO `entity_team` (`entity_id`, `entity_type`, `team_id`, `deleted`) VALUES (?, ?, ?, 0)"
);
foreach ($data['records'] as $row) {
$entityType = $row['entity_type'];
$entityId = $preserveIds ? $row['entity_id'] : ($idMapper->get($entityType, $row['entity_id']) ?? $row['entity_id']);
$teamId = $preserveIds ? $row['team_id'] : ($idMapper->get('Team', $row['team_id']) ?? $row['team_id']);
$stmt->execute([$entityId, $entityType, $teamId]);
$count++;
}
if ($io) {
$io->writeLine(" Junction entity_team: imported {$count} records");
}
}
private function readJunctionFile(string $inputDir, string $table): ?array
{
$path = rtrim($inputDir, '/') . '/_junction_' . $table . '.json';
if (!file_exists($path)) {
return null;
}
return json_decode(file_get_contents($path), true);
}
private function buildInList(array $ids): string
{
$pdo = $this->entityManager->getPDO();
return implode(',', array_map(fn($id) => $pdo->quote($id), $ids));
}
}
files/custom/Espo/Modules/DataMigration/Services/ManifestBuilder.php
+69
View File
@@ -0,0 +1,69 @@
<?php
namespace Espo\Modules\DataMigration\Services;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\Module;
class ManifestBuilder
{
private const EXTENSION_VERSION = '1.0.0';
private const TRACKED_MODULES = [
'LegalCrm',
'NetHamishpat',
'GreenInvoiceBilling',
'SmartAssistant',
'DigitalSignature',
'SmsIntegration',
];
public function __construct(
private Config $config,
private Module $module
) {}
public function build(string $outputDir, array $entityCounts, int $fileCount, int $fileSize): void
{
$modules = [];
foreach (self::TRACKED_MODULES as $mod) {
$moduleData = $this->module->get([$mod]);
if ($moduleData) {
$modules[$mod] = $moduleData['version'] ?? 'unknown';
}
}
$manifest = [
'version' => self::EXTENSION_VERSION,
'espoVersion' => $this->config->get('version', 'unknown'),
'exportedAt' => date('c'),
'sourceHost' => $this->config->get('siteUrl', 'unknown'),
'entityCounts' => $entityCounts,
'totalAttachmentFiles' => $fileCount,
'totalAttachmentSizeBytes' => $fileSize,
'modules' => $modules,
];
$path = rtrim($outputDir, '/') . '/manifest.json';
file_put_contents(
$path,
json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
);
}
public function read(string $inputDir): ?array
{
$path = rtrim($inputDir, '/') . '/manifest.json';
if (!file_exists($path)) {
return null;
}
$content = file_get_contents($path);
return json_decode($content, true);
}
}
manifest.json
+13
View File
@@ -0,0 +1,13 @@
{
"name": "DataMigration",
"version": "1.0.0",
"acceptableVersions": [
">=8.0.0"
],
"php": [
">=8.1"
],
"releaseDate": "2026-03-28",
"author": "klear",
"description": "Case data migration: export/import all case-related data between EspoCRM instances"
}