diff imrovements

diff.js

@@ -109,6 +109,7 @@ function buildUpgradePackage(versionFrom, params)
         var path = require('path');
         var fs = require('fs');
         var sys = require('util');
+        var cp = require('child_process');
 
         var version = (require('./package.json') || {}).version;
 
@@ -119,10 +120,28 @@ function buildUpgradePackage(versionFrom, params)
         var buildRelPath = 'build/EspoCRM-' + version;
         var buildPath = currentPath + '/' + buildRelPath;
         var diffFilePath = currentPath + '/build/diff';
+        var diffBeforeUpgradeFilePath = currentPath + '/build/diffBeforeUpgrade';
+
         var upgradePath = currentPath + '/build/EspoCRM-upgrade-' + acceptedVersionName + '-to-' + version;
 
+        var upgradeDataFolder = versionFrom + '-' + version;
+        var isMinorVersion = false;
+        if (versionFrom.split('.')[1] !== version.split('.')[1] || versionFrom.split('.')[0] !== version.split('.')[0]) {
+            isMinorVersion = true;
+            upgradeDataFolder = version.split('.')[0] + '.' + version.split('.')[1];
+        }
+        var upgradeDataFolderPath = currentPath + '/upgrades/' + upgradeDataFolder;
+        var upgradeFolderExists = fs.existsSync(upgradeDataFolderPath);
+
+        var upgradeData = {};
+        if (upgradeFolderExists) {
+            upgradeData = require(upgradeDataFolderPath + '/data.json') || {};
+        }
+
+        var beforeUpgradeFileList = upgradeData.beforeUpgradeFiles || [];
+
         var deleteDirRecursively = function (path) {
-            if (fs.existsSync(path)) {
+            if (fs.existsSync(path) && fs.lstatSync(path).isDirectory()) {
                 fs.readdirSync(path).forEach(function(file, index) {
                     var curPath = path + "/" + file;
                     if (fs.lstatSync(curPath).isDirectory()) {
@@ -132,10 +151,13 @@ function buildUpgradePackage(versionFrom, params)
                     }
                 });
                 fs.rmdirSync(path);
+            } else if (fs.existsSync(path) && fs.lstatSync(path).isFile()) {
+                fs.unlinkSync(path);
             }
         };
 
         deleteDirRecursively(diffFilePath);
+        deleteDirRecursively(diffBeforeUpgradeFilePath);
         deleteDirRecursively(upgradePath);
 
         execute('git rev-parse --abbrev-ref HEAD', function (branch) {
@@ -146,6 +168,9 @@ function buildUpgradePackage(versionFrom, params)
         });
 
         execute('git diff --name-only ' + versionFrom, function (stdout) {
+            if (!fs.existsSync(buildPath)) {
+                throw new Error("EspoCRM is not built. Need to run grunt before.");
+            }
 
             if (!fs.existsSync(upgradePath)) {
                 fs.mkdirSync(upgradePath);
@@ -154,8 +179,10 @@ function buildUpgradePackage(versionFrom, params)
                 fs.mkdirSync(upgradePath + '/files');
             }
 
-            if (!fs.existsSync(buildPath)) {
+            if (beforeUpgradeFileList.length) {
-                throw new Error("EspoCRM is not built. Need to run grunt before.");
+                if (!fs.existsSync(upgradePath + '/beforeUpgradeFiles')) {
+                    fs.mkdirSync(upgradePath + '/beforeUpgradeFiles');
+                }
             }
 
             process.chdir(buildPath);
@@ -178,6 +205,10 @@ function buildUpgradePackage(versionFrom, params)
 
             fs.writeFileSync(diffFilePath, fileList.join('\n'));
 
+            if (beforeUpgradeFileList.length) {
+                fs.writeFileSync(diffBeforeUpgradeFilePath, beforeUpgradeFileList.join('\n'));
+            }
+
             execute('git diff --name-only --diff-filter=D ' + versionFrom, function (stdout) {
                 var deletedFileList = [];
 
@@ -188,7 +219,11 @@ function buildUpgradePackage(versionFrom, params)
                     deletedFileList.push(file);
                 });
 
-                execute('xargs -a ' + diffFilePath + ' cp --parents -t ' + upgradePath + '/files ' , function (stdout) {
+                if (beforeUpgradeFileList.length) {
+                    cp.execSync('xargs -a ' + diffBeforeUpgradeFilePath + ' cp --parents -t ' + upgradePath + '/beforeUpgradeFiles');
+                }
+
+                execute('xargs -a ' + diffFilePath + ' cp --parents -t ' + upgradePath + '/files' , function (stdout) {
                     var d = new Date();
 
                     var monthN = ((d.getMonth() + 1).toString());
@@ -216,7 +251,7 @@ function buildUpgradePackage(versionFrom, params)
 
                         var name = acceptedVersionName+" to "+version;
 
-                        var manifest = {
+                        var manifestData = {
                             "name": "EspoCRM Upgrade "+name,
                             "type": "upgrade",
                             "version": version,
@@ -228,9 +263,19 @@ function buildUpgradePackage(versionFrom, params)
                             "delete": deletedFileList,
                         };
 
-                        fs.writeFileSync(upgradePath + '/manifest.json', JSON.stringify(manifest, null, '  '));
+                        var additionalManifestData = upgradeData.manifest || {};
+                        for (var item in additionalManifestData) {
+                            manifestData[item] = additionalManifestData[item];
+                        }
 
-                        fs.unlinkSync(diffFilePath);
+                        fs.writeFileSync(upgradePath + '/manifest.json', JSON.stringify(manifestData, null, '  '));
+
+                        if (fs.existsSync(diffFilePath)) {
+                            fs.unlinkSync(diffFilePath);
+                        }
+                        if (fs.existsSync(diffFilePath)) {
+                            fs.unlinkSync(diffBeforeUpgradeFilePath);
+                        }
 
                         console.log("Upgrade package is built: "+name+"");
 
@@ -247,4 +292,4 @@ function execute(command, callback) {
     exec(command, function(error, stdout, stderr) {
         callback(stdout);
     });
-};
+};

upgrades/5.7/data.json

@@ -0,0 +1,7 @@
+{
+    "beforeUpgradeFiles": [
+        "application/Espo/Core/Utils/Database/Helper.php"
+    ],
+    "manifest": {
+    }
+}

upgrades/5.7/scripts/AfterUpgrade.php

@@ -0,0 +1,41 @@
+<?php
+/************************************************************************
+ * This file is part of EspoCRM.
+ *
+ * EspoCRM - Open Source CRM application.
+ * Copyright (C) 2014  Yuri Kuznetsov, Taras Machyshyn, Oleksiy Avramenko
+ * Website: http://www.espocrm.com
+ *
+ * EspoCRM is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * EspoCRM is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with EspoCRM. If not, see http://www.gnu.org/licenses/.
+ ************************************************************************/
+
+class AfterUpgrade
+{
+    public function run($container)
+    {
+        $entityManager = $container->get('entityManager');
+
+        $entityManager->createEntity('ScheduledJob', [
+            'job' => 'ProcessWebhookQueue',
+            'name' => 'Process Webhook Queue',
+            'scheduling' => '*/5 * * * *',
+            'status' => 'Active',
+        ]);
+
+        $config = $container->get('config');
+
+        $config->set('hashSecretKey', \Espo\Core\Utils\Util::generateApiKey());
+        $config->save();
+    }
+}

upgrades/5.7/scripts/BeforeUpgrade.php

@@ -0,0 +1,61 @@
+<?php
+/************************************************************************
+ * This file is part of EspoCRM.
+ *
+ * EspoCRM - Open Source CRM application.
+ * Copyright (C) 2014  Yuri Kuznetsov, Taras Machyshyn, Oleksiy Avramenko
+ * Website: http://www.espocrm.com
+ *
+ * EspoCRM is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * EspoCRM is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with EspoCRM. If not, see http://www.gnu.org/licenses/.
+ ************************************************************************/
+
+class BeforeUpgrade
+{
+    public function run($container)
+    {
+        $this->container = $container;
+        $this->checkDatabaseRequirements();
+
+        $pdo = $container->get('entityManager')->getPDO();
+
+        try {
+            $pdo->query("TRUNCATE TABLE `scheduled_job_log_record`");
+        } catch (\Exception $e) {}
+    }
+
+    protected function checkDatabaseRequirements()
+    {
+        $databaseRequirements = [
+            'mysql' => '5.6.0',
+            'mariadb' => '10.0.0',
+        ];
+
+        $databaseHelper = new \Espo\Core\Utils\Database\Helper($this->container->get('config'));
+
+        $databaseType = $databaseHelper->getDatabaseType();
+        $fullVersion = $databaseHelper->getPdoDatabaseVersion($this->container->get('entityManager')->getPDO());
+
+        if (preg_match('/[0-9]+\.[0-9]+\.[0-9]+/', $fullVersion, $match)) {
+            $version = $match[0];
+            $databaseTypeLc = strtolower($databaseType);
+
+            if (isset($databaseRequirements[$databaseTypeLc])) {
+                if (version_compare($version, $databaseRequirements[$databaseTypeLc], '<')) {
+                    $msg = "Your {$databaseType} version is not supported. Please upgrade {$databaseType} to a newer version (5.6 or later).";
+                    throw new \Espo\Core\Exceptions\Error($msg);
+                }
+            }
+        }
+    }
+}

upgrades/5.8/data.json

@@ -0,0 +1,7 @@
+{
+    "manifest": {
+        "delete": [
+            "vendor/php-mime-mail-parser"
+        ]
+    }
+}

upgrades/5.8/scripts/AfterUpgrade.php

@@ -0,0 +1,66 @@
+<?php
+/************************************************************************
+ * This file is part of EspoCRM.
+ *
+ * EspoCRM - Open Source CRM application.
+ * Copyright (C) 2014  Yuri Kuznetsov, Taras Machyshyn, Oleksiy Avramenko
+ * Website: http://www.espocrm.com
+ *
+ * EspoCRM is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * EspoCRM is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with EspoCRM. If not, see http://www.gnu.org/licenses/.
+ ************************************************************************/
+
+class AfterUpgrade
+{
+    public function run($container)
+    {
+        $entityManager = $container->get('entityManager');
+
+        $this->populateOpportunityContactId($entityManager);
+    }
+
+    protected function populateOpportunityContactId($entityManager)
+    {
+        $pdo = $entityManager->getPdo();
+
+        $sql = "
+            SELECT opportunity.id AS 'opportunityId', contact.id AS `contactId` FROM `opportunity`
+            JOIN contact_opportunity ON contact_opportunity.opportunity_id = opportunity.id AND contact_opportunity.deleted = 0
+            JOIN contact ON contact.id = contact_opportunity.contact_id AND contact.deleted = 0
+            WHERE
+            contact.id IN (
+              SELECT MIN(contact.id)
+              FROM `opportunity`
+              JOIN contact_opportunity ON contact_opportunity.opportunity_id = opportunity.id AND contact_opportunity.deleted = 0
+              JOIN contact ON contact.id = contact_opportunity.contact_id AND contact.deleted = 0
+              GROUP BY opportunity.id
+            ) AND
+            opportunity.contact_id IS NULL AND
+            opportunity.deleted = 0
+        ";
+
+        $sth = $pdo->prepare($sql);
+        $sth->execute();
+
+        while ($row = $sth->fetch()) {
+            $cId = $row['contactId'] ?? null;
+            $oId = $row['opportunityId'] ?? null;
+            if (!$cId || !$oId) continue;
+
+            $q = "
+                UPDATE `opportunity` SET contact_id = ".$pdo->quote($cId)." WHERE id = ".$pdo->quote($oId)."
+            ";
+            $pdo->query($q);
+        }
+    }
+}

upgrades/5.8/scripts/BeforeUpgrade.php

@@ -0,0 +1,56 @@
+<?php
+/************************************************************************
+ * This file is part of EspoCRM.
+ *
+ * EspoCRM - Open Source CRM application.
+ * Copyright (C) 2014  Yuri Kuznetsov, Taras Machyshyn, Oleksiy Avramenko
+ * Website: http://www.espocrm.com
+ *
+ * EspoCRM is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * EspoCRM is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with EspoCRM. If not, see http://www.gnu.org/licenses/.
+ ************************************************************************/
+
+class BeforeUpgrade
+{
+    public function run($container)
+    {
+        $this->container = $container;
+        $this->checkDatabaseRequirements();
+    }
+
+    protected function checkDatabaseRequirements()
+    {
+        $databaseRequirements = [
+            'mysql' => '5.7.0',
+            'mariadb' => '10.1.0',
+        ];
+
+        $databaseHelper = new \Espo\Core\Utils\Database\Helper($this->container->get('config'));
+
+        $databaseType = $databaseHelper->getDatabaseType();
+        $fullVersion = $databaseHelper->getPdoDatabaseVersion($this->container->get('entityManager')->getPDO());
+
+        if (preg_match('/[0-9]+\.[0-9]+\.[0-9]+/', $fullVersion, $match)) {
+            $version = $match[0];
+            $databaseTypeLc = strtolower($databaseType);
+
+            if (isset($databaseRequirements[$databaseTypeLc])) {
+                $requiredVersion = $databaseRequirements[$databaseTypeLc];
+                if (version_compare($version, $requiredVersion, '<')) {
+                    $msg = "Your {$databaseType} version is not supported. Please upgrade {$databaseType} to a newer version ({$requiredVersion} or later).";
+                    throw new \Espo\Core\Exceptions\Error($msg);
+                }
+            }
+        }
+    }
+}