From d0635484a2b2319e84a6062f2259d58bcf424a89 Mon Sep 17 00:00:00 2001 From: Taras Machyshyn Date: Tue, 20 Feb 2018 15:56:34 +0200 Subject: [PATCH] Database: default column charset is utf8mb4 --- .../Database/DBAL/Platforms/MySqlPlatform.php | 16 +- .../Core/Utils/Database/Schema/Converter.php | 109 ++++++------ .../Core/Utils/Database/Schema/Schema.php | 51 +++++- .../Espo/Core/Utils/Database/Schema/Utils.php | 166 ++++++++++++++++++ application/Espo/Core/defaults/config.php | 2 +- install/core/Installer.php | 4 +- 6 files changed, 291 insertions(+), 57 deletions(-) create mode 100644 application/Espo/Core/Utils/Database/Schema/Utils.php diff --git a/application/Espo/Core/Utils/Database/DBAL/Platforms/MySqlPlatform.php b/application/Espo/Core/Utils/Database/DBAL/Platforms/MySqlPlatform.php index 699815870f..25d47cb814 100644 --- a/application/Espo/Core/Utils/Database/DBAL/Platforms/MySqlPlatform.php +++ b/application/Espo/Core/Utils/Database/DBAL/Platforms/MySqlPlatform.php @@ -133,7 +133,6 @@ class MySqlPlatform extends \Doctrine\DBAL\Platforms\MySqlPlatform ); } - return array_merge($sql, $tableSql, $columnSql); } @@ -305,10 +304,23 @@ class MySqlPlatform extends \Doctrine\DBAL\Platforms\MySqlPlatform protected function _getCreateTableSQL($tableName, array $columns, array $options = array()) { if (!isset($options['engine'])) { - $options['engine'] = 'MyISAM'; + $options['engine'] = 'InnoDB'; + } + + if (!isset($options['charset'])) { + $options['charset'] = 'utf8mb4'; + } + + if (!isset($options['collate'])) { + $options['collate'] = 'utf8mb4_unicode_ci'; } return parent::_getCreateTableSQL($tableName, $columns, $options); } + + public function getColumnCollationDeclarationSQL($collation) + { + return $this->getCollationFieldDeclaration($collation); + } //end: ESPO } \ No newline at end of file diff --git a/application/Espo/Core/Utils/Database/Schema/Converter.php b/application/Espo/Core/Utils/Database/Schema/Converter.php index d9aa1a32bd..a5dafaddd9 100644 --- a/application/Espo/Core/Utils/Database/Schema/Converter.php +++ b/application/Espo/Core/Utils/Database/Schema/Converter.php @@ -28,15 +28,18 @@ ************************************************************************/ namespace Espo\Core\Utils\Database\Schema; -use Espo\Core\Utils\Util, - Espo\ORM\Entity, - Espo\Core\Exceptions\Error; +use Espo\Core\Utils\Util; +use Espo\ORM\Entity; +use Espo\Core\Exceptions\Error; +use Espo\Core\Utils\Database\Schema\Utils as SchemaUtils; class Converter { private $dbalSchema; + private $databaseSchema; + private $fileManager; private $metadata; @@ -76,10 +79,11 @@ class Converter 'foreign' ); - public function __construct(\Espo\Core\Utils\Metadata $metadata, \Espo\Core\Utils\File\Manager $fileManager) + public function __construct(\Espo\Core\Utils\Metadata $metadata, \Espo\Core\Utils\File\Manager $fileManager, \Espo\Core\Utils\Database\Schema\Schema $databaseSchema) { $this->metadata = $metadata; $this->fileManager = $fileManager; + $this->databaseSchema = $databaseSchema; $this->typeList = array_keys(\Doctrine\DBAL\Types\Type::getTypesMap()); } @@ -110,6 +114,11 @@ class Converter return $this->dbalSchema; } + protected function getDatabaseSchema() + { + return $this->databaseSchema; + } + /** * Schema convertation process * @@ -154,6 +163,9 @@ class Converter $schema = $this->getSchema(true); + $indexList = SchemaUtils::getIndexList($ormMeta); + $fieldListExceededIndexMaxLength = SchemaUtils::getFieldListExceededIndexMaxLength($ormMeta, $this->getDatabaseSchema()->getMaxIndexLength()); + $tables = array(); foreach ($ormMeta as $entityName => $entityParams) { @@ -177,7 +189,7 @@ class Converter $primaryColumns = array(); $uniqueColumns = array(); - $indexList = array(); //list of indexes like array( array(comlumn1, column2), array(column3)) + foreach ($entityParams['fields'] as $fieldName => $fieldParams) { if ((isset($fieldParams['notStorable']) && $fieldParams['notStorable']) || in_array($fieldParams['type'], $this->notStorableTypes)) { @@ -197,37 +209,26 @@ class Converter continue; } + if (isset($fieldListExceededIndexMaxLength[$entityName]) && in_array($fieldName, $fieldListExceededIndexMaxLength[$entityName])) { + $fieldParams['utf8mb3'] = true; + } + $columnName = Util::toUnderScore($fieldName); if (!$tables[$entityName]->hasColumn($columnName)) { $tables[$entityName]->addColumn($columnName, $fieldType, $this->getDbFieldParams($fieldParams)); } //add unique - if ($fieldParams['type']!= 'id' && isset($fieldParams['unique'])) { + if ($fieldParams['type'] != 'id' && isset($fieldParams['unique'])) { $uniqueColumns = $this->getKeyList($columnName, $fieldParams['unique'], $uniqueColumns); } //END: add unique - - //add index. It can be defined in entityDefs as "index" - if (isset($fieldParams['index'])) { - $indexList = $this->getKeyList($columnName, $fieldParams['index'], $indexList); - } //END: add index } $tables[$entityName]->setPrimaryKey($primaryColumns); - //add indexes - if (isset($entityParams['indexes']) && is_array($entityParams['indexes'])) { - foreach ($entityParams['indexes'] as $indexName => $indexParams) { - if (is_array($indexParams['columns'])) { - $tableIndexName = $this->generateIndexName($indexName, $entityName); - $indexList[$tableIndexName] = Util::toUnderScore($indexParams['columns']); - } - } - } - if (!empty($indexList)) { - foreach($indexList as $indexName => $indexItem) { - $tableIndexName = is_string($indexName) ? $indexName : null; - $tables[$entityName]->addIndex($indexItem, $tableIndexName); + if (!empty($indexList[$entityName])) { + foreach($indexList[$entityName] as $indexName => $indexColumnList) { + $tables[$entityName]->addIndex($indexColumnList, $indexName); } } @@ -285,14 +286,21 @@ class Converter } $table = $this->getSchema()->createTable($tableName); - $table->addColumn('id', 'int', array('length'=>$this->defaultLength['int'], 'autoincrement' => true, 'notnull' => true,)); //'unique' => true, + $table->addColumn('id', 'int', $this->getDbFieldParams(array( + 'type' => 'int', + 'len' => $this->defaultLength['int'], + 'autoincrement' => true, + ))); //add midKeys to a schema $uniqueIndex = array(); foreach($relationParams['midKeys'] as $index => $midKey) { $columnName = Util::toUnderScore($midKey); - $table->addColumn($columnName, $this->idParams['dbType'], array('length'=>$this->idParams['len'])); + $table->addColumn($columnName, $this->idParams['dbType'], $this->getDbFieldParams(array( + 'type' => 'foreignId', + 'len' => $this->idParams['len'], + ))); $table->addIndex(array($columnName)); $uniqueIndex[] = $columnName; @@ -306,7 +314,7 @@ class Converter if (!isset($fieldParams['type'])) { $fieldParams = array_merge($fieldParams, array( 'type' => 'varchar', - 'length' => $this->defaultLength['varchar'], + 'len' => $this->defaultLength['varchar'], )); } @@ -326,7 +334,11 @@ class Converter } //END: add unique indexes - $table->addColumn('deleted', 'bool', array('default' => 0)); + $table->addColumn('deleted', 'bool', $this->getDbFieldParams(array( + 'type' => 'bool', + 'default' => false, + ))); + $table->setPrimaryKey(array("id")); return $table; @@ -337,13 +349,18 @@ class Converter $dbFieldParams = array(); foreach($this->allowedDbFieldParams as $espoName => $dbalName) { - if (isset($fieldParams[$espoName])) { $dbFieldParams[$dbalName] = $fieldParams[$espoName]; } } switch ($fieldParams['type']) { + case 'id': + case 'foreignId': + case 'foreignType': + $fieldParams['utf8mb3'] = true; + break; + case 'array': case 'jsonArray': case 'text': @@ -360,12 +377,17 @@ class Converter break; } - - if ( isset($fieldParams['autoincrement']) && $fieldParams['autoincrement'] ) { + if (isset($fieldParams['autoincrement']) && $fieldParams['autoincrement']) { $dbFieldParams['unique'] = true; $dbFieldParams['notnull'] = true; } + if (isset($fieldParams['utf8mb3']) && $fieldParams['utf8mb3']) { + $dbFieldParams['platformOptions'] = array( + 'collation' => 'utf8_unicode_ci', + ); + } + return $dbFieldParams; } @@ -378,9 +400,11 @@ class Converter protected function getKeyList($columnName, $keyValue, array $keyList) { if ($keyValue === true) { - $keyList[] = array($columnName); + $tableIndexName = SchemaUtils::generateIndexName($columnName); + $keyList[$tableIndexName] = array($columnName); } else if (is_string($keyValue)) { - $keyList[$keyValue][] = $columnName; + $tableIndexName = SchemaUtils::generateIndexName($keyValue); + $keyList[$tableIndexName][] = $columnName; } return $keyList; @@ -451,23 +475,6 @@ class Converter return $dependentEntities; } - /** - * Generate index name - * - * @return string - */ - protected function generateIndexName($name, $entityName) - { - $names = array( - 'IDX', - ); - - $names[] = strtoupper( Util::toUnderScore($entityName) ); - $names[] = strtoupper( Util::toUnderScore($name) ); - - return implode('_', $names); - } - protected function loadData($path) { $tables = array(); @@ -487,4 +494,4 @@ class Converter return $tables; } -} +} \ No newline at end of file diff --git a/application/Espo/Core/Utils/Database/Schema/Schema.php b/application/Espo/Core/Utils/Database/Schema/Schema.php index a3e4d54d6f..880244bd58 100644 --- a/application/Espo/Core/Utils/Database/Schema/Schema.php +++ b/application/Espo/Core/Utils/Database/Schema/Schema.php @@ -94,7 +94,7 @@ class Schema $this->converter = new \Espo\Core\Utils\Database\Converter($this->metadata, $this->fileManager); - $this->schemaConverter = new Converter($this->metadata, $this->fileManager); + $this->schemaConverter = new Converter($this->metadata, $this->fileManager, $this); $this->ormMetadata = $ormMetadata; } @@ -312,5 +312,54 @@ class Schema } } + public function getMaxIndexLength() + { + $connection = $this->getConnection(); + $schema = new \Espo\Core\Utils\Database\DBAL\Schema\Schema(); + $tableName = 'temp_' . uniqid(); + $table = $schema->createTable($tableName); + + $table->addColumn('id', 'varchar', array( + 'length' => 50, + )); + + $table->addColumn('column3072', 'varchar', array( + 'length' => 1020, + 'platformOptions' => array( + 'collation' => 'utf8_unicode_ci', + ), + )); + + $table->addColumn('column1000', 'varchar', array( + 'length' => 332, + 'platformOptions' => array( + 'collation' => 'utf8_unicode_ci', + ), + )); + + $table->setPrimaryKey(array("id")); + + $table->addIndex(['column3072'], 'IDX_COLUMN3072'); + $queries = $schema->toSql($this->getPlatform()); + + try { + $connection->executeQuery($queries[0]); + $connection->getSchemaManager()->dropTable($tableName); + return 3072; //InnoDB, MySQL 5.7+ + } catch (\Exception $e) {} + + $table->dropIndex('IDX_COLUMN3072'); + $table->addIndex(['column1000'], 'IDX_COLUMN1000'); + $queries = $schema->toSql($this->getPlatform()); + + try { + $connection->executeQuery($queries[0]); + $connection->getSchemaManager()->dropTable($tableName); + return 1000; //MyISAM + } catch (\Exception $e) {} + + $connection->getSchemaManager()->dropTable($tableName); + return 767; //InnoDB + } } diff --git a/application/Espo/Core/Utils/Database/Schema/Utils.php b/application/Espo/Core/Utils/Database/Schema/Utils.php new file mode 100644 index 0000000000..3e781c1686 --- /dev/null +++ b/application/Espo/Core/Utils/Database/Schema/Utils.php @@ -0,0 +1,166 @@ + $entityParams) { + foreach ($entityParams['fields'] as $fieldName => $fieldParams) { + if (isset($fieldParams['notStorable']) && $fieldParams['notStorable']) { + continue; + } + + if (isset($fieldParams['index'])) { + $keyValue = $fieldParams['index']; + $columnName = Util::toUnderScore($fieldName); + + if (!isset($indexList[$entityName])) { + $indexList[$entityName] = []; + } + + if ($keyValue === true) { + $tableIndexName = static::generateIndexName($columnName); + $indexList[$entityName][$tableIndexName] = array($columnName); + } else if (is_string($keyValue)) { + $tableIndexName = static::generateIndexName($keyValue); + $indexList[$entityName][$tableIndexName][] = $columnName; + } + } + } + + if (isset($entityParams['indexes']) && is_array($entityParams['indexes'])) { + foreach ($entityParams['indexes'] as $indexName => $indexParams) { + if (is_array($indexParams['columns'])) { + $tableIndexName = static::generateIndexName($indexName); + $indexList[$entityName][$tableIndexName] = Util::toUnderScore($indexParams['columns']); + } + } + } + } + + return $indexList; + } + + public static function generateIndexName($name, $prefix = 'IDX', $maxLength = 30) + { + $nameList = []; + $nameList[] = strtoupper($prefix); + $nameList[] = strtoupper( Util::toUnderScore($name) ); + + return substr(implode('_', $nameList), 0, $maxLength); + } + + public static function getFieldListExceededIndexMaxLength(array $ormMeta, $indexMaxLength = 1000, $characterLength = 4) + { + $permittedFieldTypeList = [ + 'varchar', + ]; + + $fields = array(); + + $indexList = static::getIndexList($ormMeta); + + foreach ($indexList as $entityName => $indexes) { + foreach ($indexes as $indexName => $columnList) { + $indexLength = 0; + foreach ($columnList as $columnName) { + $fieldName = Util::toCamelCase($columnName); + + if (!isset($ormMeta[$entityName]['fields'][$fieldName])) { + continue; + } + + $indexLength += static::getFieldLength($ormMeta[$entityName]['fields'][$fieldName], $characterLength); + } + + if ($indexLength > $indexMaxLength) { + foreach ($columnList as $columnName) { + $fieldName = Util::toCamelCase($columnName); + if (!isset($ormMeta[$entityName]['fields'][$fieldName])) { + continue; + } + + $fieldType = static::getFieldType($ormMeta[$entityName]['fields'][$fieldName]); + + if (in_array($fieldType, $permittedFieldTypeList)) { + if (!isset($fields[$entityName]) || !in_array($fieldName, $fields[$entityName])) { + $fields[$entityName][] = $fieldName; + } + } + } + } + } + } + + return $fields; + } + + protected static function getFieldLength(array $ormFieldDefs, $characterLength = 4) + { + $length = 0; + + if (isset($ormFieldDefs['notStorable']) && $ormFieldDefs['notStorable']) { + return $length; + } + + $defaultLength = array( + 'datetime' => 8, + 'time' => 4, + 'int' => 4, + 'bool' => 1, + 'float' => 4, + 'varchar' => 255, + ); + + $type = static::getFieldType($ormFieldDefs); + + $length = isset($defaultLength[$type]) ? $defaultLength[$type] : $length; + $length = isset($ormFieldDefs['len']) ? $ormFieldDefs['len'] : $length; + + switch ($type) { + case 'varchar': + $length = $length * $characterLength; + break; + } + + return $length; + } + + protected static function getFieldType(array $ormFieldDefs) + { + return isset($ormFieldDefs['dbType']) ? $ormFieldDefs['dbType'] : $ormFieldDefs['type']; + } +} diff --git a/application/Espo/Core/defaults/config.php b/application/Espo/Core/defaults/config.php index 78a2ab54cf..657763a502 100644 --- a/application/Espo/Core/defaults/config.php +++ b/application/Espo/Core/defaults/config.php @@ -32,7 +32,7 @@ return array ( 'driver' => 'pdo_mysql', 'host' => 'localhost', 'port' => '', - 'charset' => 'utf8', + 'charset' => 'utf8mb4', 'dbname' => '', 'user' => '', 'password' => '', diff --git a/install/core/Installer.php b/install/core/Installer.php index b940100477..a95f26e3c6 100644 --- a/install/core/Installer.php +++ b/install/core/Installer.php @@ -191,11 +191,11 @@ class Installer public function saveData($database, $language) { $initData = include('install/core/afterInstall/config.php'); - $siteUrl = $this->getSystemHelper()->getBaseUrl(); + $databaseDefaults = $this->app->getContainer()->get('config')->get('database'); $data = array( - 'database' => $database, + 'database' => array_merge($databaseDefaults, $database), 'language' => $language, 'siteUrl' => $siteUrl, 'passwordSalt' => $this->getPasswordHash()->generateSalt(),