From fd279b6c8b09657068b7e5028f29c9c851f318d3 Mon Sep 17 00:00:00 2001
From: Chaim Marcus <chaim@marcus-law.co.il>
Date: Sun, 29 Mar 2026 20:25:54 +0000
Subject: [PATCH] fix: skip columns missing in target DB during raw SQL import

When importing entities with autoincrement (Case, Charge, Invoice),
validate each column exists in the target table before inserting.
Columns from modules not installed on the target are silently skipped
and reported in the import log summary.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../DataMigration/Services/EntityImporter.php | 68 +++++++++++++++----
 .../DataMigration/Services/ImportService.php  | 12 ++++
 2 files changed, 67 insertions(+), 13 deletions(-)

diff --git a/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php b/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php
index 4d0c88a..9645bba 100644
--- a/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php
+++ b/files/custom/Espo/Modules/DataMigration/Services/EntityImporter.php
@@ -309,9 +309,16 @@ class EntityImporter
         return $newId;
     }
 
+    /** @var array<string, string[]> Cache of existing columns per table */
+    private array $tableColumnsCache = [];
+
+    /** @var array<string, string[]> Skipped columns log per entity type */
+    private array $skippedColumnsLog = [];
+
     private function insertWithRawSql(string $entityType, array $record, string $newId): void
     {
         $tableName = $this->camelCaseToUnderscore($entityType);
+        $existingColumns = $this->getTableColumns($tableName);
 
         // Convert camelCase fields to snake_case for SQL
         $columns = [];
@@ -325,15 +332,22 @@ class EntityImporter
                 continue;
             }
 
-            // Skip non-storable array/object fields that come from getValueMap
-            if (is_array($value) || is_object($value)) {
-                // JSON fields should be stored as JSON strings
-                if (is_array($value) || is_object($value)) {
-                    $value = json_encode($value, JSON_UNESCAPED_UNICODE);
+            $column = $this->camelCaseToUnderscore($field);
+
+            // Skip columns that don't exist in the target table
+            if (!in_array($column, $existingColumns)) {
+                if (!isset($this->skippedColumnsLog[$entityType][$column])) {
+                    $this->skippedColumnsLog[$entityType][$column] = true;
                 }
+
+                continue;
+            }
+
+            // JSON fields should be stored as JSON strings
+            if (is_array($value) || is_object($value)) {
+                $value = json_encode($value, JSON_UNESCAPED_UNICODE);
             }
 
-            $column = $this->camelCaseToUnderscore($field);
             $columns[] = "`{$column}`";
             $values[] = $value;
             $placeholders[] = '?';
@@ -344,14 +358,42 @@ class EntityImporter
 
         $sql = "INSERT INTO `{$tableName}` ({$columnStr}) VALUES ({$placeholderStr})";
 
-        try {
-            $pdo = $this->entityManager->getPDO();
-            $stmt = $pdo->prepare($sql);
-            $stmt->execute($values);
-        } catch (\Exception $e) {
-            // If column mismatch, try with just known columns
-            throw $e;
+        $pdo = $this->entityManager->getPDO();
+        $stmt = $pdo->prepare($sql);
+        $stmt->execute($values);
+    }
+
+    private function getTableColumns(string $tableName): array
+    {
+        if (isset($this->tableColumnsCache[$tableName])) {
+            return $this->tableColumnsCache[$tableName];
         }
+
+        $pdo = $this->entityManager->getPDO();
+        $sth = $pdo->query("SHOW COLUMNS FROM `{$tableName}`");
+        $columns = [];
+
+        while ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
+            $columns[] = $row['Field'];
+        }
+
+        $this->tableColumnsCache[$tableName] = $columns;
+
+        return $columns;
+    }
+
+    /**
+     * @return array<string, string[]> entityType => [skipped column names]
+     */
+    public function getSkippedColumns(): array
+    {
+        $result = [];
+
+        foreach ($this->skippedColumnsLog as $entityType => $columns) {
+            $result[$entityType] = array_keys($columns);
+        }
+
+        return $result;
     }
 
     private function resetAutoIncrement(string $entityType): void
diff --git a/files/custom/Espo/Modules/DataMigration/Services/ImportService.php b/files/custom/Espo/Modules/DataMigration/Services/ImportService.php
index 2af92d0..3305917 100644
--- a/files/custom/Espo/Modules/DataMigration/Services/ImportService.php
+++ b/files/custom/Espo/Modules/DataMigration/Services/ImportService.php
@@ -179,6 +179,18 @@ class ImportService
                 }
             }
         }
+
+        // Report skipped columns (missing in target DB)
+        $skippedColumns = $this->entityImporter->getSkippedColumns();
+
+        if (!empty($skippedColumns)) {
+            $io->writeLine('');
+            $io->writeLine('WARNING: Skipped columns not found in target database:');
+
+            foreach ($skippedColumns as $type => $columns) {
+                $io->writeLine("  {$type}: " . implode(', ', $columns));
+            }
+        }
     }
 
     private function entityTypeExists(string $entityType): bool