commit e08eca7eb33dc7321e2c87683978315f8c20599e Author: Chaim Marcus Date: Sat Mar 28 19:50:46 2026 +0000 feat: initial DataMigration extension for case data backup/restore Full case data migration tool with 3 console commands: - data-migration-export: exports all case-related entities (27 types) + files - data-migration-import: restores backup to target DB with dry-run support - data-migration-status: shows entity counts from live DB or backup Supports preserve-ids, skip-duplicates, batch processing, and attachment file copying. Imports with hooks disabled to prevent side effects. Co-Authored-By: Claude Opus 4.6 (1M context) diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..199f3fe --- /dev/null +++ b/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Build extension ZIP package +set -euo pipefail +VERSION=$(python3 -c "import json; print(json.load(open('manifest.json'))['version'])") +MODULE=$(python3 -c "import json; m=json.load(open('manifest.json')); print(m.get('module', m['name']))") +ZIPNAME="${MODULE}-${VERSION}.zip" + +# Update version in README.md if it exists +if [[ -f README.md ]]; then + sed -i "s/\*\*גרסה:\*\* [^ |]*/\*\*גרסה:\*\* ${VERSION}/" README.md +fi +echo "Building $ZIPNAME..." +rm -f "$ZIPNAME" +zip -r "$ZIPNAME" manifest.json files/ scripts/ \ + -x "*.DS_Store" "*__MACOSX*" "*.zip" 2>/dev/null || \ +zip -r "$ZIPNAME" manifest.json files/ \ + -x "*.DS_Store" "*__MACOSX*" "*.zip" +echo "✓ Built: $ZIPNAME ($(du -h "$ZIPNAME" | cut -f1))" diff --git a/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationExport.php b/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationExport.php new file mode 100644 index 0000000..9e46c5c --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationExport.php @@ -0,0 +1,78 @@ +getOption('output'); + + if (!$output) { + $timestamp = date('Ymd-His'); + $output = 'data/backups/datamigration-' . $timestamp; + } + + $output = rtrim($output, '/'); + + if (is_dir($output)) { + $io->writeLine("ERROR: Output directory already exists: {$output}"); + $io->writeLine("Please specify a new directory or remove the existing one."); + return; + } + + $batchSize = (int) ($params->getOption('batchSize') ?? 500); + $verbose = $params->hasFlag('verbose'); + $noFiles = $params->hasFlag('noFiles'); + + $caseIds = null; + $caseIdsStr = $params->getOption('caseIds'); + + if ($caseIdsStr) { + $caseIds = array_map('trim', explode(',', $caseIdsStr)); + } + + $caseStatus = null; + $statusStr = $params->getOption('status'); + + if ($statusStr) { + $caseStatus = array_map('trim', explode(',', $statusStr)); + } + + $io->writeLine('EspoCRM Data Migration - Export'); + $io->writeLine('=============================='); + $io->writeLine("Output: {$output}"); + $io->writeLine("Batch size: {$batchSize}"); + + if ($caseIds) { + $io->writeLine("Case filter: " . count($caseIds) . " specific IDs"); + } + + if ($caseStatus) { + $io->writeLine("Status filter: " . implode(', ', $caseStatus)); + } + + if ($noFiles) { + $io->writeLine("Files: SKIPPED (--no-files)"); + } + + $io->writeLine(''); + + $this->exportService->export($output, [ + 'batchSize' => $batchSize, + 'caseIds' => $caseIds, + 'caseStatus' => $caseStatus, + 'noFiles' => $noFiles, + 'verbose' => $verbose, + ], $io); + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationImport.php b/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationImport.php new file mode 100644 index 0000000..2254510 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationImport.php @@ -0,0 +1,61 @@ +getOption('input'); + + if (!$input) { + $io->writeLine('ERROR: --input option is required. Specify the backup directory path.'); + return; + } + + $input = rtrim($input, '/'); + + if (!is_dir($input)) { + $io->writeLine("ERROR: Input directory not found: {$input}"); + return; + } + + if (!file_exists($input . '/manifest.json')) { + $io->writeLine("ERROR: No manifest.json found in: {$input}"); + $io->writeLine("This does not appear to be a valid backup directory."); + return; + } + + $batchSize = (int) ($params->getOption('batchSize') ?? 500); + $dryRun = $params->hasFlag('dryRun'); + $preserveIds = $params->hasFlag('preserveIds'); + $skipDuplicates = $params->hasFlag('skipDuplicates'); + $verbose = $params->hasFlag('verbose'); + + $io->writeLine('EspoCRM Data Migration - Import'); + $io->writeLine('=============================='); + $io->writeLine("Input: {$input}"); + $io->writeLine("Batch size: {$batchSize}"); + $io->writeLine("Preserve IDs: " . ($preserveIds ? 'YES' : 'NO')); + $io->writeLine("Skip duplicates: " . ($skipDuplicates ? 'YES' : 'NO')); + $io->writeLine("Dry run: " . ($dryRun ? 'YES' : 'NO')); + $io->writeLine(''); + + $this->importService->import($input, [ + 'batchSize' => $batchSize, + 'preserveIds' => $preserveIds, + 'skipDuplicates' => $skipDuplicates, + 'dryRun' => $dryRun, + 'verbose' => $verbose, + ], $io); + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationStatus.php b/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationStatus.php new file mode 100644 index 0000000..38f0d4b --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Commands/DataMigrationStatus.php @@ -0,0 +1,167 @@ +getOption('input'); + + if ($inputDir) { + $this->showBackupStatus($inputDir, $io); + } else { + $this->showLiveStatus($io); + } + } + + private function showLiveStatus(IO $io): void + { + $io->writeLine('EspoCRM Data Migration - Live Database Status'); + $io->writeLine('============================================='); + $io->writeLine(''); + + $total = 0; + + foreach (self::ENTITY_TYPES as $entityType) { + try { + $count = $this->entityManager + ->getRDBRepository($entityType) + ->count(); + + if ($count > 0) { + $io->writeLine(sprintf(" %-30s %d", $entityType, $count)); + $total += $count; + } + } catch (\Exception $e) { + // Entity type doesn't exist (module not installed) + } + } + + $io->writeLine(''); + $io->writeLine("Total records: {$total}"); + + // Count attachments on disk + $uploadDir = 'data/upload/'; + + if (is_dir($uploadDir)) { + $fileCount = 0; + $totalSize = 0; + + $files = scandir($uploadDir); + + foreach ($files as $file) { + if ($file === '.' || $file === '..') { + continue; + } + + $path = $uploadDir . $file; + + if (is_file($path)) { + $fileCount++; + $totalSize += filesize($path); + } + } + + $io->writeLine(''); + $io->writeLine("Attachment files: {$fileCount} (" . $this->formatBytes($totalSize) . ")"); + } + } + + private function showBackupStatus(string $inputDir, IO $io): void + { + $inputDir = rtrim($inputDir, '/'); + + $io->writeLine('EspoCRM Data Migration - Backup Status'); + $io->writeLine('======================================'); + $io->writeLine("Path: {$inputDir}"); + $io->writeLine(''); + + $manifest = $this->manifestBuilder->read($inputDir); + + if ($manifest === null) { + $io->writeLine('ERROR: No manifest.json found.'); + return; + } + + $io->writeLine("Source: " . ($manifest['sourceHost'] ?? 'unknown')); + $io->writeLine("Exported at: " . ($manifest['exportedAt'] ?? 'unknown')); + $io->writeLine("EspoCRM version: " . ($manifest['espoVersion'] ?? 'unknown')); + $io->writeLine(''); + + if (!empty($manifest['modules'])) { + $io->writeLine('Modules:'); + + foreach ($manifest['modules'] as $module => $version) { + $io->writeLine(" {$module}: {$version}"); + } + + $io->writeLine(''); + } + + $io->writeLine('Entity counts:'); + + foreach ($manifest['entityCounts'] ?? [] as $type => $count) { + if ($count > 0) { + $io->writeLine(sprintf(" %-30s %d", $type, $count)); + } + } + + $io->writeLine(''); + $io->writeLine("Attachment files: " . ($manifest['totalAttachmentFiles'] ?? 0)); + $io->writeLine("Attachment size: " . $this->formatBytes($manifest['totalAttachmentSizeBytes'] ?? 0)); + } + + private function formatBytes(int $bytes): string + { + if ($bytes >= 1073741824) { + return round($bytes / 1073741824, 2) . ' GB'; + } + + if ($bytes >= 1048576) { + return round($bytes / 1048576, 2) . ' MB'; + } + + if ($bytes >= 1024) { + return round($bytes / 1024, 2) . ' KB'; + } + + return $bytes . ' B'; + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Resources/metadata/app/consoleCommands.json b/files/custom/Espo/Modules/DataMigration/Resources/metadata/app/consoleCommands.json new file mode 100644 index 0000000..7840460 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Resources/metadata/app/consoleCommands.json @@ -0,0 +1,20 @@ +{ + "dataMigrationExport": { + "className": "Espo\\Modules\\DataMigration\\Commands\\DataMigrationExport", + "listed": true, + "allowedOptions": ["output", "batchSize", "caseIds", "status"], + "allowedFlags": ["dryRun", "noFiles", "verbose"] + }, + "dataMigrationImport": { + "className": "Espo\\Modules\\DataMigration\\Commands\\DataMigrationImport", + "listed": true, + "allowedOptions": ["input", "batchSize"], + "allowedFlags": ["dryRun", "preserveIds", "skipDuplicates", "verbose"] + }, + "dataMigrationStatus": { + "className": "Espo\\Modules\\DataMigration\\Commands\\DataMigrationStatus", + "listed": true, + "allowedOptions": ["input"], + "allowedFlags": [] + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Resources/module.json b/files/custom/Espo/Modules/DataMigration/Resources/module.json new file mode 100644 index 0000000..57b2399 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Resources/module.json @@ -0,0 +1,8 @@ +{ + "order": 99, + "version": "1.0.0", + "requires": { + "espocrm": ">=8.0.0" + }, + "description": "Data migration tool for case data backup and restore" +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/AttachmentHandler.php b/files/custom/Espo/Modules/DataMigration/Services/AttachmentHandler.php new file mode 100644 index 0000000..2551869 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/AttachmentHandler.php @@ -0,0 +1,150 @@ +writeLine(" WARNING: Attachment file not found: {$id}"); + } + + continue; + } + + $destPath = $filesDir . '/' . $id; + copy($sourcePath, $destPath); + + $size = filesize($sourcePath); + $totalSize += $size; + $count++; + } + + if ($io) { + $io->writeLine(" Attachments: copied {$count} files (" . $this->formatBytes($totalSize) . ")"); + + if ($missing > 0) { + $io->writeLine(" WARNING: {$missing} attachment files not found on disk"); + } + } + + return ['count' => $count, 'size' => $totalSize]; + } + + /** + * Import attachment files from backup to data/upload/. + * + * @param string $inputDir Backup files/ directory + * @param IdMapper $idMapper + * @param bool $preserveIds + * @param bool $verbose + * @param IO|null $io + * @return int Number of files restored + */ + public function importFiles( + string $inputDir, + IdMapper $idMapper, + bool $preserveIds = true, + bool $verbose = false, + ?IO $io = null + ): int { + $filesDir = rtrim($inputDir, '/'); + + if (!is_dir($filesDir)) { + if ($io) { + $io->writeLine(" No files/ directory in backup, skipping attachment restore."); + } + + return 0; + } + + if (!is_dir(self::UPLOAD_DIR)) { + mkdir(self::UPLOAD_DIR, 0755, true); + } + + $files = scandir($filesDir); + $count = 0; + + foreach ($files as $file) { + if ($file === '.' || $file === '..') { + continue; + } + + $sourcePath = $filesDir . '/' . $file; + + if (!is_file($sourcePath)) { + continue; + } + + if ($preserveIds) { + $destId = $file; + } else { + $destId = $idMapper->get('Attachment', $file) ?? $file; + } + + $destPath = self::UPLOAD_DIR . $destId; + + if (!file_exists($destPath)) { + copy($sourcePath, $destPath); + $count++; + } + } + + if ($io) { + $io->writeLine(" Attachments: restored {$count} files"); + } + + return $count; + } + + private function formatBytes(int $bytes): string + { + if ($bytes >= 1073741824) { + return round($bytes / 1073741824, 2) . ' GB'; + } + + if ($bytes >= 1048576) { + return round($bytes / 1048576, 2) . ' MB'; + } + + if ($bytes >= 1024) { + return round($bytes / 1024, 2) . ' KB'; + } + + return $bytes . ' B'; + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/EntityExporter.php b/files/custom/Espo/Modules/DataMigration/Services/EntityExporter.php new file mode 100644 index 0000000..9678c94 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/EntityExporter.php @@ -0,0 +1,93 @@ +entityManager + ->getRDBRepository($entityType) + ->where($where) + ->limit($offset, $batchSize); + + $collection = $query->find(); + $count = 0; + + foreach ($collection as $entity) { + $record = $this->entityToArray($entity); + $records[] = $record; + $count++; + $total++; + } + + if ($count === 0) { + break; + } + + if ($verbose && $io) { + $io->writeLine(" {$entityType}: exported {$total} records..."); + } + + $offset += $batchSize; + + if ($count < $batchSize) { + break; + } + } + + $data = [ + 'entityType' => $entityType, + 'count' => $total, + 'exportedAt' => date('c'), + 'records' => $records, + ]; + + file_put_contents( + $filePath, + json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) + ); + + return $total; + } + + /** + * Convert an entity to a plain associative array using getValueMap(). + */ + private function entityToArray($entity): array + { + $valueMap = $entity->getValueMap(); + + return (array) $valueMap; + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php b/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php new file mode 100644 index 0000000..981e56d --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php @@ -0,0 +1,384 @@ + [field => referencedEntityType] + */ + private const FK_FIELDS = [ + 'Case' => [ + 'accountId' => 'Account', + 'contactId' => 'Contact', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + 'cPayerId' => 'Account', + 'cBeneficiaryId' => 'Contact', + 'cPricingAgreementId' => 'PricingAgreement', + ], + 'CaseActivity' => [ + 'caseId' => 'Case', + 'accountId' => 'Account', + 'invoiceId' => 'Invoice', + 'chargeId' => 'Charge', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'Charge' => [ + 'caseId' => 'Case', + 'accountId' => 'Account', + 'caseActivityId' => 'CaseActivity', + 'invoiceId' => 'Invoice', + 'fixedActivityRateId' => 'FixedActivityRate', + 'pricingAgreementRateId' => 'PricingAgreementRate', + 'legalAidPayerAccountId' => 'Account', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'Invoice' => [ + 'caseId' => 'Case', + 'accountId' => 'Account', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'Document' => [ + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'Note' => [ + 'parentId' => '_parent', // special: uses parentType + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'Attachment' => [ + 'parentId' => '_parent', + 'createdById' => 'User', + ], + 'NhMeeting' => ['caseId' => 'Case'], + 'NhDecision' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'], + 'NhProcess' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'], + 'NhPlea' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'], + 'NhActivity' => ['caseId' => 'Case', 'taskId' => 'Task'], + 'NhDocument' => ['caseId' => 'Case'], + 'NhSyncLog' => ['caseId' => 'Case'], + 'CaseMemory' => [ + 'caseId' => 'Case', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'SignatureRequest' => [ + 'caseId' => 'Case', + 'documentId' => 'Document', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + 'signedDocumentId' => 'Attachment', + ], + 'SignatureRequestSigner' => [ + 'signatureRequestId' => 'SignatureRequest', + 'contactId' => 'Contact', + 'userId' => 'User', + ], + 'SmsLog' => [ + 'parentId' => '_parent', + 'createdById' => 'User', + ], + 'PricingAgreement' => [ + 'accountId' => 'Account', + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + 'PricingAgreementRate' => [ + 'pricingAgreementId' => 'PricingAgreement', + ], + 'ClientRate' => [ + 'accountId' => 'Account', + ], + 'Contact' => [ + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + 'accountId' => 'Account', + ], + 'Account' => [ + 'assignedUserId' => 'User', + 'createdById' => 'User', + 'modifiedById' => 'User', + ], + ]; + + /** + * Fields to skip during import (non-storable, computed, etc.). + */ + private const SKIP_FIELDS = [ + 'deleted', + 'totalBillableAmount', + 'unbilledAmount', + 'contactRole', + 'contactPartyOrder', + 'caseRole', + 'casePartyOrder', + 'cLastActivityIndicator', + 'signerCount', + 'signedCount', + ]; + + /** + * Entity types that have an autoincrement 'number' field. + */ + private const AUTOINCREMENT_ENTITIES = ['Case', 'Charge', 'Invoice']; + + public function __construct( + private EntityManager $entityManager, + private Metadata $metadata + ) {} + + /** + * Import a single entity type from its JSON file. + * + * @return array{imported: int, skipped: int, errors: int} + */ + public function import( + string $entityType, + string $jsonFilePath, + bool $preserveIds, + bool $skipDuplicates, + IdMapper $idMapper, + int $batchSize, + bool $verbose = false, + bool $dryRun = false, + ?IO $io = null + ): array { + if (!file_exists($jsonFilePath)) { + return ['imported' => 0, 'skipped' => 0, 'errors' => 0]; + } + + $content = json_decode(file_get_contents($jsonFilePath), true); + + if (!$content || empty($content['records'])) { + return ['imported' => 0, 'skipped' => 0, 'errors' => 0]; + } + + $records = $content['records']; + $imported = 0; + $skipped = 0; + $errors = 0; + $hasAutoIncrement = in_array($entityType, self::AUTOINCREMENT_ENTITIES); + + foreach ($records as $record) { + $oldId = $record['id'] ?? null; + + if (!$oldId) { + $errors++; + continue; + } + + // Check for duplicates + if ($preserveIds) { + $existing = $this->entityManager + ->getRDBRepository($entityType) + ->where(['id' => $oldId]) + ->findOne(); + + if ($existing) { + if ($skipDuplicates) { + $skipped++; + $idMapper->add($entityType, $oldId, $oldId); + continue; + } + + $errors++; + + if ($verbose && $io) { + $io->writeLine(" ERROR: {$entityType} ID {$oldId} already exists"); + } + + continue; + } + } + + if ($dryRun) { + $newId = $preserveIds ? $oldId : $idMapper->generateId(); + $idMapper->add($entityType, $oldId, $newId); + $imported++; + continue; + } + + try { + $newId = $this->importRecord($entityType, $record, $preserveIds, $idMapper, $hasAutoIncrement); + $idMapper->add($entityType, $oldId, $newId); + $imported++; + } catch (\Exception $e) { + $errors++; + + if ($verbose && $io) { + $io->writeLine(" ERROR importing {$entityType} {$oldId}: " . $e->getMessage()); + } + } + } + + // Reset autoincrement counter + if (!$dryRun && $hasAutoIncrement && $imported > 0) { + $this->resetAutoIncrement($entityType); + } + + return ['imported' => $imported, 'skipped' => $skipped, 'errors' => $errors]; + } + + private function importRecord( + string $entityType, + array $record, + bool $preserveIds, + IdMapper $idMapper, + bool $hasAutoIncrement + ): string { + $oldId = $record['id']; + $newId = $preserveIds ? $oldId : $idMapper->generateId(); + + // Remap foreign keys if not preserving IDs + if (!$preserveIds) { + $fkMap = self::FK_FIELDS[$entityType] ?? []; + + foreach ($fkMap as $field => $refType) { + if (!isset($record[$field]) || $record[$field] === null) { + continue; + } + + if ($refType === '_parent') { + // Polymorphic: use parentType to determine entity type + $parentTypeField = str_replace('Id', 'Type', $field); + $parentType = $record[$parentTypeField] ?? null; + + if ($parentType) { + $mapped = $idMapper->get($parentType, $record[$field]); + + if ($mapped) { + $record[$field] = $mapped; + } + } + } else { + $mapped = $idMapper->get($refType, $record[$field]); + + if ($mapped) { + $record[$field] = $mapped; + } + } + } + } + + // For entities with autoincrement number, use raw SQL + if ($hasAutoIncrement) { + $this->insertWithRawSql($entityType, $record, $newId); + + return $newId; + } + + // Use ORM for standard entities + $entity = $this->entityManager->getRDBRepository($entityType)->getNew(); + $entity->set('id', $newId); + + foreach ($record as $field => $value) { + if ($field === 'id' || in_array($field, self::SKIP_FIELDS)) { + continue; + } + + $entity->set($field, $value); + } + + $this->entityManager->saveEntity($entity, [ + 'silent' => true, + 'skipHooks' => true, + 'noStream' => true, + 'skipAll' => true, + ]); + + return $newId; + } + + private function insertWithRawSql(string $entityType, array $record, string $newId): void + { + $tableName = $this->entityManager + ->getQueryComposer() + ->toDb($entityType); + + // Convert camelCase fields to snake_case for SQL + $columns = []; + $values = []; + $placeholders = []; + + $record['id'] = $newId; + + foreach ($record as $field => $value) { + if (in_array($field, self::SKIP_FIELDS)) { + continue; + } + + // Skip non-storable array/object fields that come from getValueMap + if (is_array($value) || is_object($value)) { + // JSON fields should be stored as JSON strings + if (is_array($value) || is_object($value)) { + $value = json_encode($value, JSON_UNESCAPED_UNICODE); + } + } + + $column = $this->camelCaseToUnderscore($field); + $columns[] = "`{$column}`"; + $values[] = $value; + $placeholders[] = '?'; + } + + $columnStr = implode(', ', $columns); + $placeholderStr = implode(', ', $placeholders); + + $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})"; + + try { + $pdo = $this->entityManager->getPDO(); + $stmt = $pdo->prepare($sql); + $stmt->execute($values); + } catch (\Exception $e) { + // If column mismatch, try with just known columns + throw $e; + } + } + + private function resetAutoIncrement(string $entityType): void + { + $tableName = $this->entityManager + ->getQueryComposer() + ->toDb($entityType); + + try { + $pdo = $this->entityManager->getPDO(); + $sql = "SELECT MAX(`number`) as max_num FROM `{$tableName}`"; + $sth = $pdo->query($sql); + $row = $sth->fetch(\PDO::FETCH_ASSOC); + + if ($row && $row['max_num']) { + $next = (int) $row['max_num'] + 1; + $pdo->exec("ALTER TABLE `{$tableName}` AUTO_INCREMENT = {$next}"); + } + } catch (\Exception $e) { + // Non-critical, log and continue + } + } + + private function camelCaseToUnderscore(string $str): string + { + return strtolower(preg_replace('/(?writeLine('Starting export...'); + $io->writeLine(''); + + // === Determine Case IDs === + $caseWhere = []; + + if ($filterCaseIds) { + $caseWhere['id'] = $filterCaseIds; + } + + if ($filterStatus) { + $caseWhere['status'] = $filterStatus; + } + + $caseIds = $this->collectIds('Case', $caseWhere); + $entityIdsByType['Case'] = $caseIds; + $io->writeLine("Found " . count($caseIds) . " cases to export."); + + if (empty($caseIds)) { + $io->writeLine('No cases found. Nothing to export.'); + return; + } + + // === Layer 0: Users, Teams, FixedActivityRate === + $io->writeLine(''); + $io->writeLine('[Layer 0] Reference entities...'); + + $referencedUserIds = $this->collectReferencedUserIds($caseIds); + $entityIdsByType['User'] = $referencedUserIds; + + if (!empty($referencedUserIds)) { + $entityCounts['User'] = $this->entityExporter->export( + 'User', ['id' => $referencedUserIds], $dataDir, $batchSize, $verbose, $io + ); + } + + $referencedTeamIds = $this->collectReferencedTeamIds($caseIds); + $entityIdsByType['Team'] = $referencedTeamIds; + + if (!empty($referencedTeamIds)) { + $entityCounts['Team'] = $this->entityExporter->export( + 'Team', ['id' => $referencedTeamIds], $dataDir, $batchSize, $verbose, $io + ); + } + + if ($this->entityTypeExists('FixedActivityRate')) { + $entityCounts['FixedActivityRate'] = $this->entityExporter->export( + 'FixedActivityRate', [], $dataDir, $batchSize, $verbose, $io + ); + } + + // === Layer 1: Accounts and Contacts === + $io->writeLine(''); + $io->writeLine('[Layer 1] Accounts and Contacts...'); + + $accountIds = $this->collectReferencedAccountIds($caseIds); + $entityIdsByType['Account'] = $accountIds; + + if (!empty($accountIds)) { + $entityCounts['Account'] = $this->entityExporter->export( + 'Account', ['id' => $accountIds], $dataDir, $batchSize, $verbose, $io + ); + } + + $contactIds = $this->collectReferencedContactIds($caseIds); + $entityIdsByType['Contact'] = $contactIds; + + if (!empty($contactIds)) { + $entityCounts['Contact'] = $this->entityExporter->export( + 'Contact', ['id' => $contactIds], $dataDir, $batchSize, $verbose, $io + ); + } + + // === Layer 2: Billing rates === + $io->writeLine(''); + $io->writeLine('[Layer 2] Billing rates...'); + + if ($this->entityTypeExists('PricingAgreement') && !empty($accountIds)) { + $entityCounts['PricingAgreement'] = $this->entityExporter->export( + 'PricingAgreement', ['accountId' => $accountIds], $dataDir, $batchSize, $verbose, $io + ); + + $paIds = $this->collectIds('PricingAgreement', ['accountId' => $accountIds]); + $entityIdsByType['PricingAgreement'] = $paIds; + + if (!empty($paIds) && $this->entityTypeExists('PricingAgreementRate')) { + $entityCounts['PricingAgreementRate'] = $this->entityExporter->export( + 'PricingAgreementRate', ['pricingAgreementId' => $paIds], $dataDir, $batchSize, $verbose, $io + ); + } + } + + if ($this->entityTypeExists('ClientRate') && !empty($accountIds)) { + $entityCounts['ClientRate'] = $this->entityExporter->export( + 'ClientRate', ['accountId' => $accountIds], $dataDir, $batchSize, $verbose, $io + ); + } + + // === Layer 3: Cases === + $io->writeLine(''); + $io->writeLine('[Layer 3] Cases...'); + + $entityCounts['Case'] = $this->entityExporter->export( + 'Case', ['id' => $caseIds], $dataDir, $batchSize, $verbose, $io + ); + + // === Layer 4: Direct Case children === + $io->writeLine(''); + $io->writeLine('[Layer 4] Case children...'); + + $layer4Entities = [ + 'CaseActivity', 'Invoice', 'NhMeeting', 'NhDecision', + 'NhProcess', 'NhPlea', 'NhActivity', 'NhDocument', + 'NhSyncLog', 'CaseMemory', 'SignatureRequest', + ]; + + foreach ($layer4Entities as $entityType) { + if (!$this->entityTypeExists($entityType)) { + continue; + } + + $entityCounts[$entityType] = $this->entityExporter->export( + $entityType, ['caseId' => $caseIds], $dataDir, $batchSize, $verbose, $io + ); + + $entityIdsByType[$entityType] = $this->collectIds($entityType, ['caseId' => $caseIds]); + } + + // Document - linked via junction table + if ($this->entityTypeExists('Document')) { + $documentIds = $this->collectDocumentIdsForCases($caseIds); + $entityIdsByType['Document'] = $documentIds; + + if (!empty($documentIds)) { + $entityCounts['Document'] = $this->entityExporter->export( + 'Document', ['id' => $documentIds], $dataDir, $batchSize, $verbose, $io + ); + } + } + + // === Layer 5: Second-level children === + $io->writeLine(''); + $io->writeLine('[Layer 5] Second-level children...'); + + if ($this->entityTypeExists('Charge')) { + $entityCounts['Charge'] = $this->entityExporter->export( + 'Charge', ['caseId' => $caseIds], $dataDir, $batchSize, $verbose, $io + ); + $entityIdsByType['Charge'] = $this->collectIds('Charge', ['caseId' => $caseIds]); + } + + if ($this->entityTypeExists('SignatureRequestSigner')) { + $srIds = $entityIdsByType['SignatureRequest'] ?? []; + + if (!empty($srIds)) { + $entityCounts['SignatureRequestSigner'] = $this->entityExporter->export( + 'SignatureRequestSigner', ['signatureRequestId' => $srIds], $dataDir, $batchSize, $verbose, $io + ); + } + } + + if ($this->entityTypeExists('SmsLog')) { + $entityCounts['SmsLog'] = $this->entityExporter->export( + 'SmsLog', ['parentType' => 'Case', 'parentId' => $caseIds], $dataDir, $batchSize, $verbose, $io + ); + } + + // === Layer 6: Notes and Attachments === + $io->writeLine(''); + $io->writeLine('[Layer 6] Notes and Attachments...'); + + $entityCounts['Note'] = $this->entityExporter->export( + 'Note', ['parentType' => 'Case', 'parentId' => $caseIds], $dataDir, $batchSize, $verbose, $io + ); + + $attachmentIds = $this->collectAllAttachmentIds($dataDir); + $entityIdsByType['Attachment'] = $attachmentIds; + + if (!empty($attachmentIds)) { + $entityCounts['Attachment'] = $this->entityExporter->export( + 'Attachment', ['id' => $attachmentIds], $dataDir, $batchSize, $verbose, $io + ); + } + + // === Junction tables === + $io->writeLine(''); + $io->writeLine('[Junctions] Relationship tables...'); + $this->junctionTableHandler->exportJunctions($caseIds, $entityIdsByType, $dataDir, $io); + + // === Files === + $fileCount = 0; + $fileSize = 0; + + if (!$noFiles && !empty($attachmentIds)) { + $io->writeLine(''); + $io->writeLine('[Files] Copying attachment files...'); + $result = $this->attachmentHandler->exportFiles($attachmentIds, $filesDir, $verbose, $io); + $fileCount = $result['count']; + $fileSize = $result['size']; + } + + // === Manifest === + $this->manifestBuilder->build($outputDir, $entityCounts, $fileCount, $fileSize); + + $io->writeLine(''); + $io->writeLine('=== Export Complete ==='); + $io->writeLine("Output: {$outputDir}"); + + foreach ($entityCounts as $type => $count) { + if ($count > 0) { + $io->writeLine(" {$type}: {$count}"); + } + } + } + + private function collectIds(string $entityType, array $where): array + { + $ids = []; + $collection = $this->entityManager + ->getRDBRepository($entityType) + ->select(['id']) + ->where($where) + ->find(); + + foreach ($collection as $entity) { + $ids[] = $entity->getId(); + } + + return $ids; + } + + private function collectReferencedUserIds(array $caseIds): array + { + if (empty($caseIds)) { + return []; + } + + $userIds = []; + $pdo = $this->entityManager->getPDO(); + + $inList = $this->buildInList($caseIds); + + // From Cases - all user reference fields + $sql = "SELECT DISTINCT assigned_user_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 + UNION + SELECT DISTINCT created_by_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 + UNION + SELECT DISTINCT modified_by_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0"; + + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + if ($row[0]) { + $userIds[$row[0]] = true; + } + } + + // From child entities + $childTables = ['case_activity', 'charge', 'invoice']; + + foreach ($childTables as $table) { + if (!$this->tableExists($table)) { + continue; + } + + $sql = "SELECT DISTINCT assigned_user_id FROM `{$table}` WHERE case_id IN ({$inList}) AND deleted = 0 + UNION + SELECT DISTINCT created_by_id FROM `{$table}` WHERE case_id IN ({$inList}) AND deleted = 0"; + + try { + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + if ($row[0]) { + $userIds[$row[0]] = true; + } + } + } catch (\Exception $e) { + // Table may not have these columns + } + } + + return array_keys($userIds); + } + + private function collectReferencedTeamIds(array $caseIds): array + { + if (empty($caseIds)) { + return []; + } + + $pdo = $this->entityManager->getPDO(); + $inList = $this->buildInList($caseIds); + + $sql = "SELECT DISTINCT team_id FROM `entity_team` WHERE entity_type = 'Case' AND entity_id IN ({$inList}) AND deleted = 0"; + $sth = $pdo->query($sql); + + $teamIds = []; + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + if ($row[0]) { + $teamIds[] = $row[0]; + } + } + + return $teamIds; + } + + private function collectReferencedAccountIds(array $caseIds): array + { + if (empty($caseIds)) { + return []; + } + + $pdo = $this->entityManager->getPDO(); + $inList = $this->buildInList($caseIds); + $accountIds = []; + + $sql = "SELECT DISTINCT account_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 AND account_id IS NOT NULL"; + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + $accountIds[$row[0]] = true; + } + + // cPayerId from GreenInvoiceBilling + try { + $sql = "SELECT DISTINCT c_payer_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 AND c_payer_id IS NOT NULL"; + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + $accountIds[$row[0]] = true; + } + } catch (\Exception $e) { + // Column may not exist + } + + return array_keys($accountIds); + } + + private function collectReferencedContactIds(array $caseIds): array + { + if (empty($caseIds)) { + return []; + } + + $pdo = $this->entityManager->getPDO(); + $inList = $this->buildInList($caseIds); + $contactIds = []; + + $sql = "SELECT DISTINCT contact_id FROM `case` WHERE id IN ({$inList}) AND deleted = 0 AND contact_id IS NOT NULL"; + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + $contactIds[$row[0]] = true; + } + + $sql = "SELECT DISTINCT contact_id FROM `case_contact` WHERE case_id IN ({$inList}) AND deleted = 0"; + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + $contactIds[$row[0]] = true; + } + + return array_keys($contactIds); + } + + private function collectDocumentIdsForCases(array $caseIds): array + { + if (empty($caseIds)) { + return []; + } + + $pdo = $this->entityManager->getPDO(); + $inList = $this->buildInList($caseIds); + + $sql = "SELECT DISTINCT document_id FROM `case_document` WHERE case_id IN ({$inList}) AND deleted = 0"; + $sth = $pdo->query($sql); + + $ids = []; + + while ($row = $sth->fetch(PDO::FETCH_NUM)) { + $ids[] = $row[0]; + } + + return $ids; + } + + private function collectAllAttachmentIds(string $dataDir): array + { + $attachmentIds = []; + $files = glob($dataDir . '/*.json'); + + foreach ($files as $file) { + $basename = basename($file); + + if (str_starts_with($basename, '_junction_')) { + continue; + } + + $content = json_decode(file_get_contents($file), true); + + if (!$content || empty($content['records'])) { + continue; + } + + foreach ($content['records'] as $record) { + // Known file fields + $knownFileFields = ['fileId', 'signedDocumentId']; + + foreach ($knownFileFields as $ff) { + if (!empty($record[$ff])) { + $attachmentIds[$record[$ff]] = true; + } + } + + // attachmentsIds (attachmentMultiple) + if (!empty($record['attachmentsIds']) && is_array($record['attachmentsIds'])) { + foreach ($record['attachmentsIds'] as $attId) { + $attachmentIds[$attId] = true; + } + } + + // Note data field + if (isset($record['data'])) { + $data = is_array($record['data']) ? $record['data'] : (array) $record['data']; + + if (!empty($data['attachmentsIds']) && is_array($data['attachmentsIds'])) { + foreach ($data['attachmentsIds'] as $attId) { + $attachmentIds[$attId] = true; + } + } + } + } + } + + return array_keys($attachmentIds); + } + + /** + * Build a quoted IN list from an array of IDs. + */ + private function buildInList(array $ids): string + { + $pdo = $this->entityManager->getPDO(); + + return implode(',', array_map(fn($id) => $pdo->quote($id), $ids)); + } + + private function entityTypeExists(string $entityType): bool + { + try { + $this->entityManager->getRDBRepository($entityType); + return true; + } catch (\Exception $e) { + return false; + } + } + + private function tableExists(string $table): bool + { + try { + $this->entityManager->getSqlExecutor()->execute("SELECT 1 FROM `{$table}` LIMIT 1"); + return true; + } catch (\Exception $e) { + return false; + } + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/IdMapper.php b/files/custom/Espo/Modules/DataMigration/Services/IdMapper.php new file mode 100644 index 0000000..e3f2477 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/IdMapper.php @@ -0,0 +1,80 @@ +> [entityType][oldId] => newId */ + private array $map = []; + + public function add(string $entityType, string $oldId, string $newId): void + { + $this->map[$entityType][$oldId] = $newId; + } + + public function get(string $entityType, ?string $oldId): ?string + { + if ($oldId === null) { + return null; + } + + return $this->map[$entityType][$oldId] ?? null; + } + + public function has(string $entityType, string $oldId): bool + { + return isset($this->map[$entityType][$oldId]); + } + + /** + * Try to resolve an ID across multiple entity types. + * Useful for polymorphic parent fields. + */ + public function resolve(?string $oldId): ?string + { + if ($oldId === null) { + return null; + } + + foreach ($this->map as $entries) { + if (isset($entries[$oldId])) { + return $entries[$oldId]; + } + } + + return null; + } + + public function generateId(): string + { + return Util::generateId(); + } + + public function getMap(): array + { + return $this->map; + } + + public function saveTo(string $path): void + { + file_put_contents( + $path, + json_encode($this->map, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) + ); + } + + public function loadFrom(string $path): void + { + if (!file_exists($path)) { + return; + } + + $data = json_decode(file_get_contents($path), true); + + if (is_array($data)) { + $this->map = $data; + } + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/ImportService.php b/files/custom/Espo/Modules/DataMigration/Services/ImportService.php new file mode 100644 index 0000000..4f6ef73 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/ImportService.php @@ -0,0 +1,174 @@ +manifestBuilder->read($inputDir); + + if ($manifest === null) { + $io->writeLine('ERROR: No manifest.json found in backup directory.'); + return; + } + + $io->writeLine('=== Backup Manifest ==='); + $io->writeLine("Source: " . ($manifest['sourceHost'] ?? 'unknown')); + $io->writeLine("Exported at: " . ($manifest['exportedAt'] ?? 'unknown')); + $io->writeLine("EspoCRM version: " . ($manifest['espoVersion'] ?? 'unknown')); + $io->writeLine(''); + + if ($dryRun) { + $io->writeLine('*** DRY RUN MODE - No changes will be made ***'); + $io->writeLine(''); + } + + $idMapper = new IdMapper(); + $totalImported = 0; + $totalSkipped = 0; + $totalErrors = 0; + $results = []; + + // Import entities in order + foreach (self::IMPORT_ORDER as $entityType) { + $jsonFile = $dataDir . '/' . $entityType . '.json'; + + if (!file_exists($jsonFile)) { + continue; + } + + $io->writeLine("Importing {$entityType}..."); + + $result = $this->entityImporter->import( + $entityType, + $jsonFile, + $preserveIds, + $skipDuplicates, + $idMapper, + $batchSize, + $verbose, + $dryRun, + $io + ); + + $results[$entityType] = $result; + $totalImported += $result['imported']; + $totalSkipped += $result['skipped']; + $totalErrors += $result['errors']; + + $io->writeLine(" {$entityType}: imported={$result['imported']}, skipped={$result['skipped']}, errors={$result['errors']}"); + } + + // Import junction tables + $io->writeLine(''); + $io->writeLine('Importing junction tables...'); + + if (!$dryRun) { + $this->junctionTableHandler->importJunctions($dataDir, $idMapper, $preserveIds, $io); + } else { + $io->writeLine(' (skipped in dry-run mode)'); + } + + // Restore attachment files + if (is_dir($filesDir)) { + $io->writeLine(''); + $io->writeLine('Restoring attachment files...'); + + if (!$dryRun) { + $this->attachmentHandler->importFiles($filesDir, $idMapper, $preserveIds, $verbose, $io); + } else { + $fileCount = count(array_diff(scandir($filesDir), ['.', '..'])); + $io->writeLine(" Would restore {$fileCount} files (dry-run)"); + } + } + + // Save ID mapping for reference + if (!$dryRun && !$preserveIds) { + $mapPath = rtrim($inputDir, '/') . '/id-mapping.json'; + $idMapper->saveTo($mapPath); + $io->writeLine(''); + $io->writeLine("ID mapping saved to: {$mapPath}"); + } + + // Summary + $io->writeLine(''); + $io->writeLine('=== Import ' . ($dryRun ? 'Dry Run ' : '') . 'Complete ==='); + $io->writeLine("Total imported: {$totalImported}"); + $io->writeLine("Total skipped: {$totalSkipped}"); + $io->writeLine("Total errors: {$totalErrors}"); + + if ($totalErrors > 0) { + $io->writeLine(''); + $io->writeLine('Entities with errors:'); + + foreach ($results as $type => $r) { + if ($r['errors'] > 0) { + $io->writeLine(" {$type}: {$r['errors']} errors"); + } + } + } + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/JunctionTableHandler.php b/files/custom/Espo/Modules/DataMigration/Services/JunctionTableHandler.php new file mode 100644 index 0000000..8a264d3 --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/JunctionTableHandler.php @@ -0,0 +1,231 @@ +entityManager->getPDO(); + + // case_contact + $inList = $this->buildInList($caseIds); + $counts['case_contact'] = $this->exportTableQuery( + $pdo, + 'case_contact', + "SELECT * FROM `case_contact` WHERE `case_id` IN ({$inList}) AND `deleted` = 0", + $outputDir + ); + + // case_document + $counts['case_document'] = $this->exportTableQuery( + $pdo, + 'case_document', + "SELECT * FROM `case_document` WHERE `case_id` IN ({$inList}) AND `deleted` = 0", + $outputDir + ); + + // entity_team - for all tracked entity types + $allTeamRecords = []; + + foreach (self::TEAM_ENTITY_TYPES as $entityType) { + $ids = $entityIdsByType[$entityType] ?? []; + + if (empty($ids)) { + continue; + } + + $entityInList = $this->buildInList($ids); + $quotedType = $pdo->quote($entityType); + + $sql = "SELECT * FROM `entity_team` WHERE `entity_type` = {$quotedType} AND `entity_id` IN ({$entityInList}) AND `deleted` = 0"; + $sth = $pdo->query($sql); + + while ($row = $sth->fetch(PDO::FETCH_ASSOC)) { + $allTeamRecords[] = $row; + } + } + + $filePath = rtrim($outputDir, '/') . '/_junction_entity_team.json'; + file_put_contents( + $filePath, + json_encode([ + 'table' => 'entity_team', + 'count' => count($allTeamRecords), + 'exportedAt' => date('c'), + 'records' => $allTeamRecords, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) + ); + $counts['entity_team'] = count($allTeamRecords); + + if ($io) { + foreach ($counts as $table => $count) { + $io->writeLine(" Junction {$table}: {$count} records"); + } + } + + return $counts; + } + + /** + * Import junction table records from backup. + */ + public function importJunctions( + string $inputDir, + IdMapper $idMapper, + bool $preserveIds = true, + ?IO $io = null + ): void { + $this->importCaseContact($inputDir, $idMapper, $preserveIds, $io); + $this->importCaseDocument($inputDir, $idMapper, $preserveIds, $io); + $this->importEntityTeam($inputDir, $idMapper, $preserveIds, $io); + } + + private function exportTableQuery(PDO $pdo, string $table, string $sql, string $outputDir): int + { + $sth = $pdo->query($sql); + $records = $sth->fetchAll(PDO::FETCH_ASSOC); + + $filePath = rtrim($outputDir, '/') . '/_junction_' . $table . '.json'; + file_put_contents( + $filePath, + json_encode([ + 'table' => $table, + 'count' => count($records), + 'exportedAt' => date('c'), + 'records' => $records, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) + ); + + return count($records); + } + + private function importCaseContact(string $inputDir, IdMapper $idMapper, bool $preserveIds, ?IO $io): void + { + $data = $this->readJunctionFile($inputDir, 'case_contact'); + + if ($data === null) { + return; + } + + $pdo = $this->entityManager->getPDO(); + $count = 0; + + $stmt = $pdo->prepare( + "INSERT IGNORE INTO `case_contact` (`case_id`, `contact_id`, `role`, `party_order`, `deleted`) VALUES (?, ?, ?, ?, 0)" + ); + + foreach ($data['records'] as $row) { + $caseId = $preserveIds ? $row['case_id'] : ($idMapper->get('Case', $row['case_id']) ?? $row['case_id']); + $contactId = $preserveIds ? $row['contact_id'] : ($idMapper->get('Contact', $row['contact_id']) ?? $row['contact_id']); + + $stmt->execute([$caseId, $contactId, $row['role'] ?? null, $row['party_order'] ?? null]); + $count++; + } + + if ($io) { + $io->writeLine(" Junction case_contact: imported {$count} records"); + } + } + + private function importCaseDocument(string $inputDir, IdMapper $idMapper, bool $preserveIds, ?IO $io): void + { + $data = $this->readJunctionFile($inputDir, 'case_document'); + + if ($data === null) { + return; + } + + $pdo = $this->entityManager->getPDO(); + $count = 0; + + $stmt = $pdo->prepare( + "INSERT IGNORE INTO `case_document` (`case_id`, `document_id`, `deleted`) VALUES (?, ?, 0)" + ); + + foreach ($data['records'] as $row) { + $caseId = $preserveIds ? $row['case_id'] : ($idMapper->get('Case', $row['case_id']) ?? $row['case_id']); + $documentId = $preserveIds ? $row['document_id'] : ($idMapper->get('Document', $row['document_id']) ?? $row['document_id']); + + $stmt->execute([$caseId, $documentId]); + $count++; + } + + if ($io) { + $io->writeLine(" Junction case_document: imported {$count} records"); + } + } + + private function importEntityTeam(string $inputDir, IdMapper $idMapper, bool $preserveIds, ?IO $io): void + { + $data = $this->readJunctionFile($inputDir, 'entity_team'); + + if ($data === null) { + return; + } + + $pdo = $this->entityManager->getPDO(); + $count = 0; + + $stmt = $pdo->prepare( + "INSERT IGNORE INTO `entity_team` (`entity_id`, `entity_type`, `team_id`, `deleted`) VALUES (?, ?, ?, 0)" + ); + + foreach ($data['records'] as $row) { + $entityType = $row['entity_type']; + $entityId = $preserveIds ? $row['entity_id'] : ($idMapper->get($entityType, $row['entity_id']) ?? $row['entity_id']); + $teamId = $preserveIds ? $row['team_id'] : ($idMapper->get('Team', $row['team_id']) ?? $row['team_id']); + + $stmt->execute([$entityId, $entityType, $teamId]); + $count++; + } + + if ($io) { + $io->writeLine(" Junction entity_team: imported {$count} records"); + } + } + + private function readJunctionFile(string $inputDir, string $table): ?array + { + $path = rtrim($inputDir, '/') . '/_junction_' . $table . '.json'; + + if (!file_exists($path)) { + return null; + } + + return json_decode(file_get_contents($path), true); + } + + private function buildInList(array $ids): string + { + $pdo = $this->entityManager->getPDO(); + + return implode(',', array_map(fn($id) => $pdo->quote($id), $ids)); + } +} diff --git a/files/custom/Espo/Modules/DataMigration/Services/ManifestBuilder.php b/files/custom/Espo/Modules/DataMigration/Services/ManifestBuilder.php new file mode 100644 index 0000000..325a5ce --- /dev/null +++ b/files/custom/Espo/Modules/DataMigration/Services/ManifestBuilder.php @@ -0,0 +1,69 @@ +module->get([$mod]); + + if ($moduleData) { + $modules[$mod] = $moduleData['version'] ?? 'unknown'; + } + } + + $manifest = [ + 'version' => self::EXTENSION_VERSION, + 'espoVersion' => $this->config->get('version', 'unknown'), + 'exportedAt' => date('c'), + 'sourceHost' => $this->config->get('siteUrl', 'unknown'), + 'entityCounts' => $entityCounts, + 'totalAttachmentFiles' => $fileCount, + 'totalAttachmentSizeBytes' => $fileSize, + 'modules' => $modules, + ]; + + $path = rtrim($outputDir, '/') . '/manifest.json'; + + file_put_contents( + $path, + json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) + ); + } + + public function read(string $inputDir): ?array + { + $path = rtrim($inputDir, '/') . '/manifest.json'; + + if (!file_exists($path)) { + return null; + } + + $content = file_get_contents($path); + + return json_decode($content, true); + } +} diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000..2542036 --- /dev/null +++ b/manifest.json @@ -0,0 +1,13 @@ +{ + "name": "DataMigration", + "version": "1.0.0", + "acceptableVersions": [ + ">=8.0.0" + ], + "php": [ + ">=8.1" + ], + "releaseDate": "2026-03-28", + "author": "klear", + "description": "Case data migration: export/import all case-related data between EspoCRM instances" +}