From 6a791713ea97649fbb23a09eeada45bac5adf742 Mon Sep 17 00:00:00 2001
From: Yuri Kuznetsov <yurikuzn@users.noreply.github.com>
Date: Wed, 16 Jun 2021 12:43:26 +0300
Subject: [PATCH] job group processing

---
 .../Espo/Core/Job/AbstractGroupJob.php        | 135 ++++++++++++++++++
 application/Espo/Jobs/ProcessJobGroup.php     |  34 +++++
 .../Resources/metadata/app/scheduledJobs.json |   3 +
 .../metadata/entityDefs/ScheduledJob.json     |   5 +
 tests/integration/Espo/Core/Job/JobTest.php   |   4 +-
 5 files changed, 178 insertions(+), 3 deletions(-)
 create mode 100644 application/Espo/Core/Job/AbstractGroupJob.php
 create mode 100644 application/Espo/Jobs/ProcessJobGroup.php

diff --git a/application/Espo/Core/Job/AbstractGroupJob.php b/application/Espo/Core/Job/AbstractGroupJob.php
new file mode 100644
index 0000000000..b42edb9122
--- /dev/null
+++ b/application/Espo/Core/Job/AbstractGroupJob.php
@@ -0,0 +1,135 @@
+<?php
+/************************************************************************
+ * This file is part of EspoCRM.
+ *
+ * EspoCRM - Open Source CRM application.
+ * Copyright (C) 2014-2021 Yurii Kuznietsov, Taras Machyshyn, Oleksii Avramenko
+ * Website: https://www.espocrm.com
+ *
+ * EspoCRM is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * EspoCRM is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with EspoCRM. If not, see http://www.gnu.org/licenses/.
+ *
+ * The interactive user interfaces in modified source and object code versions
+ * of this program must display Appropriate Legal Notices, as required under
+ * Section 5 of the GNU General Public License version 3.
+ *
+ * In accordance with Section 7(b) of the GNU General Public License version 3,
+ * these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
+ ************************************************************************/
+
+namespace Espo\Core\Job;
+
+use Espo\Core\Utils\DateTime;
+use Espo\Core\Utils\Config;
+
+use Espo\ORM\EntityManager;
+
+use Espo\Entities\Job as JobEntity;
+
+use DateTimeImmutable;
+
+abstract class AbstractGroupJob implements JobPreparable
+{
+    private const PORTION_NUMBER = 100;
+
+    private $jobManager;
+
+    private $entityManager;
+
+    private $config;
+
+    public function __construct(
+        JobManager $jobManager,
+        EntityManager $entityManager,
+        Config $config
+    ) {
+        $this->jobManager = $jobManager;
+        $this->entityManager = $entityManager;
+        $this->config = $config;
+    }
+
+    public function run(JobData $data): void
+    {
+        $limit = $this->config->get('jobGroupMaxPortion') ?? self::PORTION_NUMBER;
+
+        $group = $data->get('group');
+
+        $this->jobManager->processGroup($group, $limit);
+    }
+
+    public function prepare(ScheduledJobData $data, DateTimeImmutable $executeTime): void
+    {
+        $groupList = [];
+
+        $query = $this->entityManager
+            ->getQueryBuilder()
+            ->select('group')
+            ->from(JobEntity::ENTITY_TYPE)
+            ->where([
+                'status' => JobStatus::PENDING,
+                'queue' => null,
+                'group!=' => null,
+                'executeTime<=' => $executeTime->format(DateTime::SYSTEM_DATE_TIME_FORMAT),
+            ])
+            ->groupBy('group')
+            ->build();
+
+        $sth = $this->entityManager->getQueryExecutor()->execute($query);
+
+        while ($row = $sth->fetch()) {
+            $group = $row['group'];
+
+            if ($group === null) {
+                continue;
+            }
+
+            $groupList[] = $group;
+        }
+
+        if (!count($groupList)) {
+            return;
+        }
+
+        foreach ($groupList as $group) {
+            $existingJob = $this->entityManager
+                ->getRDBRepository(JobEntity::ENTITY_TYPE)
+                ->select('id')
+                ->where([
+                    'scheduledJobId' => $data->getId(),
+                    'targetGroup' => $group,
+                    'status' => [
+                        JobStatus::RUNNING,
+                        JobStatus::READY,
+                        JobStatus::PENDING,
+                    ],
+                ])
+                ->findOne();
+
+            if ($existingJob) {
+                continue;
+            }
+
+            $name = $data->getName() . ' :: ' . $group;
+
+            $this->entityManager->createEntity(JobEntity::ENTITY_TYPE, [
+                'scheduledJobId' => $data->getId(),
+                'executeTime' => $executeTime->format(DateTime::SYSTEM_DATE_TIME_FORMAT),
+                'name' => $name,
+                'data' => [
+                    'group' => $group,
+                ],
+                'targetGroup' => $group,
+            ]);
+        }
+    }
+}
diff --git a/application/Espo/Jobs/ProcessJobGroup.php b/application/Espo/Jobs/ProcessJobGroup.php
new file mode 100644
index 0000000000..99519c3aed
--- /dev/null
+++ b/application/Espo/Jobs/ProcessJobGroup.php
@@ -0,0 +1,34 @@
+<?php
+/************************************************************************
+ * This file is part of EspoCRM.
+ *
+ * EspoCRM - Open Source CRM application.
+ * Copyright (C) 2014-2021 Yurii Kuznietsov, Taras Machyshyn, Oleksii Avramenko
+ * Website: https://www.espocrm.com
+ *
+ * EspoCRM is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * EspoCRM is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with EspoCRM. If not, see http://www.gnu.org/licenses/.
+ *
+ * The interactive user interfaces in modified source and object code versions
+ * of this program must display Appropriate Legal Notices, as required under
+ * Section 5 of the GNU General Public License version 3.
+ *
+ * In accordance with Section 7(b) of the GNU General Public License version 3,
+ * these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
+ ************************************************************************/
+
+namespace Espo\Jobs;
+
+use Espo\Core\Job\AbstractGroupJob;
+
+class ProcessJobGroup extends AbstractGroupJob {}
diff --git a/application/Espo/Resources/metadata/app/scheduledJobs.json b/application/Espo/Resources/metadata/app/scheduledJobs.json
index c082f114db..eae7a8797a 100644
--- a/application/Espo/Resources/metadata/app/scheduledJobs.json
+++ b/application/Espo/Resources/metadata/app/scheduledJobs.json
@@ -1,4 +1,7 @@
 {
+    "ProcessJobGroup": {
+        "isPreparable": true
+    },
     "CheckEmailAccounts": {
         "isPreparable": true
     },
diff --git a/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json b/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json
index 2901341eea..447e242b0c 100644
--- a/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json
+++ b/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json
@@ -92,6 +92,11 @@
             "isSystem": true,
             "scheduling": "25 5 * * *"
         },
+        "ProcessJobGroup": {
+            "name": "Process Job Group",
+            "isSystem": true,
+            "scheduling": "* * * * *"
+        },
         "ProcessJobQueueQ0": {
             "name": "Process Job Queue q0",
             "isSystem": true,
diff --git a/tests/integration/Espo/Core/Job/JobTest.php b/tests/integration/Espo/Core/Job/JobTest.php
index c4f320ba58..ceced76f77 100644
--- a/tests/integration/Espo/Core/Job/JobTest.php
+++ b/tests/integration/Espo/Core/Job/JobTest.php
@@ -89,9 +89,7 @@ class JobTest extends \tests\integration\Core\BaseTestCase
             'group' => 'group-1',
         ]);
 
-        // @todo Change to `$this->jobManager->process();`.
-        $this->jobManager->processGroup('group-0', 100);
-        $this->jobManager->processGroup('group-1', 100);
+        $this->jobManager->process();
 
         $job1Reloaded = $this->entityManager->getEntity('Job', $job1->getId());
         $job2Reloaded = $this->entityManager->getEntity('Job', $job2->getId());