DataMigration/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php

<?php

namespace Espo\Modules\DataMigration\Services;

use Espo\ORM\EntityManager;
use Espo\Core\Console\IO;
use Espo\Core\Utils\Metadata;
use PDO;

class EntityImporter
{
    /**
     * Fields to remap as foreign keys per entity type.
     * Format: entityType => [field => referencedEntityType]
     */
    private const FK_FIELDS = [
        'Case' => [
            'accountId' => 'Account',
            'contactId' => 'Contact',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
            'cPayerId' => 'Account',
            'cBeneficiaryId' => 'Contact',
            'cPricingAgreementId' => 'PricingAgreement',
        ],
        'CaseActivity' => [
            'caseId' => 'Case',
            'accountId' => 'Account',
            'invoiceId' => 'Invoice',
            'chargeId' => 'Charge',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'Charge' => [
            'caseId' => 'Case',
            'accountId' => 'Account',
            'caseActivityId' => 'CaseActivity',
            'invoiceId' => 'Invoice',
            'fixedActivityRateId' => 'FixedActivityRate',
            'pricingAgreementRateId' => 'PricingAgreementRate',
            'legalAidPayerAccountId' => 'Account',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'Invoice' => [
            'caseId' => 'Case',
            'accountId' => 'Account',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'Document' => [
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'Note' => [
            'parentId' => '_parent',  // special: uses parentType
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'Attachment' => [
            'parentId' => '_parent',
            'createdById' => 'User',
        ],
        'NhMeeting' => ['caseId' => 'Case'],
        'NhDecision' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'],
        'NhProcess' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'],
        'NhPlea' => ['caseId' => 'Case', 'nhDocumentId' => 'NhDocument'],
        'NhActivity' => ['caseId' => 'Case', 'taskId' => 'Task'],
        'NhDocument' => ['caseId' => 'Case'],
        'NhSyncLog' => ['caseId' => 'Case'],
        'CaseMemory' => [
            'caseId' => 'Case',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'SignatureRequest' => [
            'caseId' => 'Case',
            'documentId' => 'Document',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
            'signedDocumentId' => 'Attachment',
        ],
        'SignatureRequestSigner' => [
            'signatureRequestId' => 'SignatureRequest',
            'contactId' => 'Contact',
            'userId' => 'User',
        ],
        'SmsLog' => [
            'parentId' => '_parent',
            'createdById' => 'User',
        ],
        'PricingAgreement' => [
            'accountId' => 'Account',
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
        'PricingAgreementRate' => [
            'pricingAgreementId' => 'PricingAgreement',
        ],
        'ClientRate' => [
            'accountId' => 'Account',
        ],
        'Contact' => [
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
            'accountId' => 'Account',
        ],
        'Account' => [
            'assignedUserId' => 'User',
            'createdById' => 'User',
            'modifiedById' => 'User',
        ],
    ];

    /**
     * Fields to skip during import (non-storable, computed, etc.).
     */
    private const SKIP_FIELDS = [
        'deleted',
        'totalBillableAmount',
        'unbilledAmount',
        'contactRole',
        'contactPartyOrder',
        'caseRole',
        'casePartyOrder',
        'cLastActivityIndicator',
        'signerCount',
        'signedCount',
    ];

    /**
     * Entity types that have an autoincrement 'number' field.
     */
    private const AUTOINCREMENT_ENTITIES = ['Case', 'Charge', 'Invoice'];

    public function __construct(
        private EntityManager $entityManager,
        private Metadata $metadata
    ) {}

    /**
     * Import a single entity type from its JSON file.
     *
     * @return array{imported: int, skipped: int, errors: int}
     */
    public function import(
        string $entityType,
        string $jsonFilePath,
        bool $preserveIds,
        bool $skipDuplicates,
        IdMapper $idMapper,
        int $batchSize,
        bool $verbose = false,
        bool $dryRun = false,
        ?IO $io = null
    ): array {
        if (!file_exists($jsonFilePath)) {
            return ['imported' => 0, 'skipped' => 0, 'errors' => 0];
        }

        $content = json_decode(file_get_contents($jsonFilePath), true);

        if (!$content || empty($content['records'])) {
            return ['imported' => 0, 'skipped' => 0, 'errors' => 0];
        }

        $records = $content['records'];
        $imported = 0;
        $skipped = 0;
        $errors = 0;
        $hasAutoIncrement = in_array($entityType, self::AUTOINCREMENT_ENTITIES);

        foreach ($records as $record) {
            $oldId = $record['id'] ?? null;

            if (!$oldId) {
                $errors++;
                continue;
            }

            // Check for duplicates
            if ($preserveIds) {
                $existing = $this->entityManager
                    ->getRDBRepository($entityType)
                    ->where(['id' => $oldId])
                    ->findOne();

                if ($existing) {
                    if ($skipDuplicates) {
                        $skipped++;
                        $idMapper->add($entityType, $oldId, $oldId);
                        continue;
                    }

                    $errors++;

                    if ($verbose && $io) {
                        $io->writeLine("  ERROR: {$entityType} ID {$oldId} already exists");
                    }

                    continue;
                }
            }

            if ($dryRun) {
                $newId = $preserveIds ? $oldId : $idMapper->generateId();
                $idMapper->add($entityType, $oldId, $newId);
                $imported++;
                continue;
            }

            try {
                $newId = $this->importRecord($entityType, $record, $preserveIds, $idMapper, $hasAutoIncrement);
                $idMapper->add($entityType, $oldId, $newId);
                $imported++;
            } catch (\Exception $e) {
                $errors++;

                if ($verbose && $io) {
                    $io->writeLine("  ERROR importing {$entityType} {$oldId}: " . $e->getMessage());
                }
            }
        }

        // Reset autoincrement counter
        if (!$dryRun && $hasAutoIncrement && $imported > 0) {
            $this->resetAutoIncrement($entityType);
        }

        return ['imported' => $imported, 'skipped' => $skipped, 'errors' => $errors];
    }

    private function importRecord(
        string $entityType,
        array $record,
        bool $preserveIds,
        IdMapper $idMapper,
        bool $hasAutoIncrement
    ): string {
        $oldId = $record['id'];
        $newId = $preserveIds ? $oldId : $idMapper->generateId();

        // Remap foreign keys if not preserving IDs
        if (!$preserveIds) {
            $fkMap = self::FK_FIELDS[$entityType] ?? [];

            foreach ($fkMap as $field => $refType) {
                if (!isset($record[$field]) || $record[$field] === null) {
                    continue;
                }

                if ($refType === '_parent') {
                    // Polymorphic: use parentType to determine entity type
                    $parentTypeField = str_replace('Id', 'Type', $field);
                    $parentType = $record[$parentTypeField] ?? null;

                    if ($parentType) {
                        $mapped = $idMapper->get($parentType, $record[$field]);

                        if ($mapped) {
                            $record[$field] = $mapped;
                        }
                    }
                } else {
                    $mapped = $idMapper->get($refType, $record[$field]);

                    if ($mapped) {
                        $record[$field] = $mapped;
                    }
                }
            }
        }

        // For entities with autoincrement number, use raw SQL
        if ($hasAutoIncrement) {
            $this->insertWithRawSql($entityType, $record, $newId);

            return $newId;
        }

        // Use ORM for standard entities
        $entity = $this->entityManager->getRDBRepository($entityType)->getNew();
        $entity->set('id', $newId);

        foreach ($record as $field => $value) {
            if ($field === 'id' || in_array($field, self::SKIP_FIELDS)) {
                continue;
            }

            $entity->set($field, $value);
        }

        $this->entityManager->saveEntity($entity, [
            'silent' => true,
            'skipHooks' => true,
            'noStream' => true,
            'skipAll' => true,
        ]);

        return $newId;
    }

    /** @var array<string, string[]> Cache of existing column names per table */
    private array $tableColumnsCache = [];

    /** @var array<string, array<string, array>> Cache of column metadata per table */
    private array $tableColumnMetaCache = [];

    /** @var array<string, string[]> Skipped columns log per entity type */
    private array $skippedColumnsLog = [];

    private function insertWithRawSql(string $entityType, array $record, string $newId): void
    {
        $tableName = $this->camelCaseToUnderscore($entityType);
        $existingColumns = $this->getTableColumns($tableName);

        // Convert camelCase fields to snake_case for SQL
        $columns = [];
        $values = [];
        $placeholders = [];

        $record['id'] = $newId;

        foreach ($record as $field => $value) {
            if (in_array($field, self::SKIP_FIELDS)) {
                continue;
            }

            $column = $this->camelCaseToUnderscore($field);

            // Skip columns that don't exist in the target table
            if (!in_array($column, $existingColumns)) {
                if (!isset($this->skippedColumnsLog[$entityType][$column])) {
                    $this->skippedColumnsLog[$entityType][$column] = true;
                }

                continue;
            }

            // JSON fields should be stored as JSON strings
            if (is_array($value) || is_object($value)) {
                $value = json_encode($value, JSON_UNESCAPED_UNICODE);
            }

            // Coerce value to match target column type
            $value = $this->coerceValue($tableName, $column, $value);

            $columns[] = "`{$column}`";
            $values[] = $value;
            $placeholders[] = '?';
        }

        $columnStr = implode(', ', $columns);
        $placeholderStr = implode(', ', $placeholders);

        $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";

        $pdo = $this->entityManager->getPDO();
        $stmt = $pdo->prepare($sql);
        $stmt->execute($values);
    }

    private function getTableColumns(string $tableName): array
    {
        if (isset($this->tableColumnsCache[$tableName])) {
            return $this->tableColumnsCache[$tableName];
        }

        $pdo = $this->entityManager->getPDO();
        $sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
        $columns = [];
        $meta = [];

        while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
            $columns[] = $row['Field'];
            $meta[$row['Field']] = [
                'type' => $row['Type'],
                'nullable' => $row['Null'] === 'YES',
                'default' => $row['Default'],
            ];
        }

        $this->tableColumnsCache[$tableName] = $columns;
        $this->tableColumnMetaCache[$tableName] = $meta;

        return $columns;
    }

    /**
     * Coerce a value to be compatible with the target column type.
     */
    private function coerceValue(string $tableName, string $column, mixed $value): mixed
    {
        $meta = $this->tableColumnMetaCache[$tableName][$column] ?? null;

        if ($meta === null) {
            return $value;
        }

        $type = strtolower($meta['type']);
        $isNumeric = str_contains($type, 'int') || str_contains($type, 'decimal')
            || str_contains($type, 'float') || str_contains($type, 'double')
            || str_contains($type, 'tinyint') || str_contains($type, 'smallint')
            || str_contains($type, 'bigint');

        if ($isNumeric && ($value === '' || $value === false)) {
            if ($meta['nullable']) {
                return null;
            }

            return 0;
        }

        if (!$meta['nullable'] && $value === null) {
            if ($isNumeric) {
                return 0;
            }

            return $meta['default'] ?? '';
        }

        return $value;
    }

    /**
     * @return array<string, string[]> entityType => [skipped column names]
     */
    public function getSkippedColumns(): array
    {
        $result = [];

        foreach ($this->skippedColumnsLog as $entityType => $columns) {
            $result[$entityType] = array_keys($columns);
        }

        return $result;
    }

    private function resetAutoIncrement(string $entityType): void
    {
        $tableName = $this->entityManager
            ->getQueryComposer()
            ->toDb($entityType);

        try {
            $pdo = $this->entityManager->getPDO();
            $sql = "SELECT MAX(`number`) as max_num FROM `{$tableName}`";
            $sth = $pdo->query($sql);
            $row = $sth->fetch(\PDO::FETCH_ASSOC);

            if ($row && $row['max_num']) {
                $next = (int) $row['max_num'] + 1;
                $pdo->exec("ALTER TABLE `{$tableName}` AUTO_INCREMENT = {$next}");
            }
        } catch (\Exception $e) {
            // Non-critical, log and continue
        }
    }

    private function camelCaseToUnderscore(string $str): string
    {
        return strtolower(preg_replace('/(?<!^)[A-Z]/', '_$0', $str));
    }
}