diff --git a/application/Espo/Classes/Cleanup/Exports.php b/application/Espo/Classes/Cleanup/Exports.php new file mode 100644 index 0000000000..0818641a62 --- /dev/null +++ b/application/Espo/Classes/Cleanup/Exports.php @@ -0,0 +1,73 @@ +config = $config; + $this->entityManager = $entityManager; + } + + public function process(): void + { + $period = '-' . $this->config->get('cleanupExportsPeriod', $this->cleanupPeriod); + + $before = DateTime::createNow() + ->modify($period) + ->getString(); + + $delete = $this->entityManager + ->getQueryBuilder() + ->delete() + ->from(Export::ENTITY_TYPE) + ->where([ + 'createdAt<' => $before, + ]) + ->build(); + + $this->entityManager->getQueryExecutor()->execute($delete); + } +} diff --git a/application/Espo/Controllers/Export.php b/application/Espo/Controllers/Export.php index ba1053bbcd..4b6b7f9026 100644 --- a/application/Espo/Controllers/Export.php +++ b/application/Espo/Controllers/Export.php @@ -29,17 +29,15 @@ namespace Espo\Controllers; -use Espo\Core\{ - Api\Request, - Exceptions\BadRequest, -}; +use Espo\Core\Api\Request; +use Espo\Core\Api\Response; +use Espo\Core\Exceptions\BadRequest; -use Espo\Tools\Export\{ - Service, - Params, -}; +use Espo\Tools\Export\Service; +use Espo\Tools\Export\ServiceParams; +use Espo\Tools\Export\Params; -use StdClass; +use stdClass; class Export { @@ -50,17 +48,52 @@ class Export $this->service = $service; } - public function postActionProcess(Request $request): StdClass + public function postActionProcess(Request $request): stdClass { $params = $this->fetchRawParamsFromRequest($request); - $result = $this->service->process($params); + $serviceParams = ServiceParams::create() + ->withIsIdle( + $request->getParsedBody()->idle ?? false + ); + + $result = $this->service->process($params, $serviceParams); + + if ($result->hasResult()) { + return (object) [ + 'id' => $result->getResult()->getAttachmentId(), + ]; + } return (object) [ - 'id' => $result->getAttachmentId(), + 'exportId' => $result->getId(), ]; } + public function getActionStatus(Request $request): stdClass + { + $id = $request->getQueryParam('id'); + + if (!$id) { + throw new BadRequest(); + } + + return $this->service->getStatusData($id); + } + + public function postActionSubscribeToNotificationOnSuccess(Request $request, Response $response): void + { + $id = $request->getParsedBody()->id ?? null; + + if (!$id || !is_string($id)) { + throw new BadRequest(); + } + + $this->service->subscribeToNotificationOnSuccess($id); + + $response->writeBody('true'); + } + private function fetchRawParamsFromRequest(Request $request): Params { $data = $request->getParsedBody(); diff --git a/application/Espo/Entities/Export.php b/application/Espo/Entities/Export.php new file mode 100644 index 0000000000..48caa98824 --- /dev/null +++ b/application/Espo/Entities/Export.php @@ -0,0 +1,130 @@ +get('params'); + + if (!is_string($raw)) { + throw new Error("No 'params'."); + } + + /** @var Params $params */ + $params = unserialize($raw); + + return $params; + } + + public function getStatus(): string + { + $value = $this->get('status'); + + if (!is_string($value)) { + throw new Error("No 'status'."); + } + + return $value; + } + + public function getAttachmentId(): ?string + { + /** @var ?string */ + return $this->get('attachmentId'); + } + + public function notifyOnFinish(): bool + { + return (bool) $this->get('notifyOnFinish'); + } + + public function getCreatedAt(): DateTime + { + $value = $this->getValueObject('createdAt'); + + if (!$value instanceof DateTime) { + throw new Error("No 'createdAt'."); + } + + return $value; + } + + public function getCreatedBy(): Link + { + $value = $this->getValueObject('createdBy'); + + if (!$value instanceof Link) { + throw new Error("No 'createdBy'."); + } + + return $value; + } + + public function setStatus(string $status): self + { + $this->set('status', $status); + + return $this; + } + + public function setAttachmentId(string $attachmentId): self + { + $this->set('attachmentId', $attachmentId); + + return $this; + } + + public function setNotifyOnFinish(bool $notifyOnFinish = true): self + { + $this->set('notifyOnFinish', $notifyOnFinish); + + return $this; + } +} diff --git a/application/Espo/Resources/defaults/systemConfig.php b/application/Espo/Resources/defaults/systemConfig.php index 05ec109748..8b78588a17 100644 --- a/application/Espo/Resources/defaults/systemConfig.php +++ b/application/Espo/Resources/defaults/systemConfig.php @@ -291,4 +291,5 @@ return [ 'ldapPortalUserLdapAuth' => false, 'passwordGenerateLength' => 10, 'massActionIdleCountThreshold' => 100, + 'exportIdleCountThreshold' => 1000, ]; diff --git a/application/Espo/Resources/i18n/en_US/Export.json b/application/Espo/Resources/i18n/en_US/Export.json index 0bbc938150..c0105e92bd 100644 --- a/application/Espo/Resources/i18n/en_US/Export.json +++ b/application/Espo/Resources/i18n/en_US/Export.json @@ -2,12 +2,23 @@ "fields": { "exportAllFields": "Export all fields", "fieldList": "Field List", - "format": "Format" + "format": "Format", + "status": "Status" }, "options": { "format": { "csv": "CSV", "xlsx": "XLSX (Excel)" + }, + "status": { + "Pending": "Pending", + "Running": "Running", + "Success": "Success", + "Failed": "Failed" } + }, + "messages": { + "exportProcessed": "Export has been processed. Download the [file]({url}).", + "infoText": "The export is being processed in idle by cron. It can take some time to finish. Closing this modal dialog won't affect the execution process." } } diff --git a/application/Espo/Resources/i18n/en_US/Global.json b/application/Espo/Resources/i18n/en_US/Global.json index 99746fc6a2..9b46549a64 100644 --- a/application/Espo/Resources/i18n/en_US/Global.json +++ b/application/Espo/Resources/i18n/en_US/Global.json @@ -265,7 +265,8 @@ "Save & New": "Save & New", "Field": "Field", "Resolution": "Resolution", - "Resolve Conflict": "Resolve Conflict" + "Resolve Conflict": "Resolve Conflict", + "Download": "Download" }, "messages": { "pleaseWait": "Please wait...", diff --git a/application/Espo/Resources/metadata/app/cleanup.json b/application/Espo/Resources/metadata/app/cleanup.json index 3e9da79df6..ca9254626a 100644 --- a/application/Espo/Resources/metadata/app/cleanup.json +++ b/application/Espo/Resources/metadata/app/cleanup.json @@ -12,5 +12,8 @@ }, "massActions": { "className": "Espo\\Classes\\Cleanup\\MassActions" + }, + "exports": { + "className": "Espo\\Classes\\Cleanup\\Exports" } } diff --git a/application/Espo/Resources/metadata/entityDefs/Export.json b/application/Espo/Resources/metadata/entityDefs/Export.json new file mode 100644 index 0000000000..3f624a9cf2 --- /dev/null +++ b/application/Espo/Resources/metadata/entityDefs/Export.json @@ -0,0 +1,34 @@ +{ + "fields": { + "status": { + "type": "enum", + "options": ["Pending", "Running", "Success", "Failed"], + "default": "Pending" + }, + "params": { + "type": "text" + }, + "createdAt": { + "type": "datetime", + "readOnly": true + }, + "createdBy": { + "type": "link", + "required": true + }, + "notifyOnFinish": { + "type": "bool", + "default": false + }, + "attachment": { + "type": "link", + "entity": "Attachment" + } + }, + "links": { + "createdBy": { + "type": "belongsTo", + "entity": "User" + } + } +} diff --git a/application/Espo/Resources/metadata/scopes/Export.json b/application/Espo/Resources/metadata/scopes/Export.json new file mode 100644 index 0000000000..89c3030cc6 --- /dev/null +++ b/application/Espo/Resources/metadata/scopes/Export.json @@ -0,0 +1,3 @@ +{ + "languageIsGlobal": true +} diff --git a/application/Espo/Tools/Export/Factory.php b/application/Espo/Tools/Export/Factory.php index d25508760d..41aa4c0307 100644 --- a/application/Espo/Tools/Export/Factory.php +++ b/application/Espo/Tools/Export/Factory.php @@ -29,21 +29,38 @@ namespace Espo\Tools\Export; -use Espo\Core\{ - InjectableFactory, -}; +use Espo\Core\InjectableFactory; +use Espo\Entities\User; + +use Espo\Core\AclManager; +use Espo\Core\Acl; + +use Espo\Core\Binding\BindingContainerBuilder; class Factory { - private $injectableFactory; + private InjectableFactory $injectableFactory; - public function __construct(InjectableFactory $injectableFactory) + private AclManager $aclManager; + + public function __construct(InjectableFactory $injectableFactory, AclManager $aclManager) { $this->injectableFactory = $injectableFactory; + $this->aclManager = $aclManager; } public function create(): Export { return $this->injectableFactory->create(Export::class); } + + public function createForUser(User $user): Export + { + $bindingContainer = BindingContainerBuilder::create() + ->bindInstance(User::class, $user) + ->bindInstance(Acl::class, $this->aclManager->createUserAcl($user)) + ->build(); + + return $this->injectableFactory->createWithBinding(Export::class, $bindingContainer); + } } diff --git a/application/Espo/Tools/Export/Jobs/Process.php b/application/Espo/Tools/Export/Jobs/Process.php new file mode 100644 index 0000000000..04c4d89160 --- /dev/null +++ b/application/Espo/Tools/Export/Jobs/Process.php @@ -0,0 +1,157 @@ +entityManager = $entityManager; + $this->factory = $factory; + $this->language = $language; + } + + public function run(JobData $data): void + { + $id = $data->getTargetId(); + + if ($id === null) { + throw new Error("ID not passed to the mass action job."); + } + + /** @var ExportEntity|null $entity */ + $entity = $this->entityManager->getEntity(ExportEntity::ENTITY_TYPE, $id); + + if ($entity === null) { + throw new Error("Export '{$id}' not found."); + } + + /** @var User|null $user */ + $user = $this->entityManager->getEntity(User::ENTITY_TYPE, $entity->getCreatedBy()->getId()); + + if (!$user) { + throw new Error("Export entity '{$id}', user not found."); + } + + try { + $export = $this->factory->createForUser($user); + + $this->setRunning($entity); + + $result = $export + ->setParams($entity->getParams()) + ->run(); + } + catch (Throwable $e) { + $this->setFailed($entity); + + throw new Error("Export job error: " . $e->getMessage()); + } + + $this->setSuccess($entity, $result); + + $this->entityManager->refreshEntity($entity); + + if ($entity->notifyOnFinish()) { + $this->notifyFinish($entity); + } + } + + private function notifyFinish(ExportEntity $entity): void + { + /** @var Notification $notification */ + $notification = $this->entityManager->getNewEntity(Notification::ENTITY_TYPE); + + $url = '?entryPoint=download&id=' . $entity->getAttachmentId(); + + $message = str_replace( + '{url}', + $url, + $this->language->translate('exportProcessed', 'messages', 'Export') + ); + + $notification + ->setType(Notification::TYPE_MESSAGE) + ->setMessage($message) + ->setUserId($entity->getCreatedBy()->getId()); + + $this->entityManager->saveEntity($notification); + } + + private function setFailed(ExportEntity $entity): void + { + $entity->setStatus(ExportEntity::STATUS_FAILED); + + $this->entityManager->saveEntity($entity); + } + + private function setRunning(ExportEntity $entity): void + { + $entity->setStatus(ExportEntity::STATUS_RUNNING); + + $this->entityManager->saveEntity($entity); + } + + private function setSuccess(ExportEntity $entity, Result $result): void + { + $entity + ->setStatus(ExportEntity::STATUS_SUCCESS) + ->setAttachmentId($result->getAttachmentId()); + + $this->entityManager->saveEntity($entity); + } +} diff --git a/application/Espo/Tools/Export/Service.php b/application/Espo/Tools/Export/Service.php index 2f89e87114..aecd927389 100644 --- a/application/Espo/Tools/Export/Service.php +++ b/application/Espo/Tools/Export/Service.php @@ -29,43 +29,60 @@ namespace Espo\Tools\Export; -use Espo\Core\{ - Exceptions\ForbiddenSilent, - Acl, - Acl\Table, - Utils\Config, - Utils\Metadata, -}; +use Espo\Core\Exceptions\ForbiddenSilent; +use Espo\Core\Exceptions\NotFoundSilent; +use Espo\Core\Acl; +use Espo\Core\Acl\Table; +use Espo\Core\Utils\Config; +use Espo\Core\Utils\Metadata; +use Espo\Core\Job\JobSchedulerFactory; +use Espo\Core\Job\Job\Data as JobData; + +use Espo\Tools\Export\Jobs\Process; + +use Espo\ORM\EntityManager; + +use Espo\Entities\Export as ExportEntity; use Espo\Entities\User; +use stdClass; + class Service { - private $factory; + private Factory $factory; - private $config; + private Config $config; - private $acl; + private Acl $acl; - private $user; + private User $user; - private $metadata; + private Metadata $metadata; + + private EntityManager $entityManager; + + private JobSchedulerFactory $jobSchedulerFactory; public function __construct( Factory $factory, Config $config, Acl $acl, User $user, - Metadata $metadata + Metadata $metadata, + EntityManager $entityManager, + JobSchedulerFactory $jobSchedulerFactory ) { $this->factory = $factory; $this->config = $config; $this->acl = $acl; $this->user = $user; $this->metadata = $metadata; + $this->entityManager = $entityManager; + $this->jobSchedulerFactory = $jobSchedulerFactory; } - public function process(Params $params): Result + public function process(Params $params, ServiceParams $serviceParams): ServiceResult { if ($this->config->get('exportDisabled') && !$this->user->isAdmin()) { throw new ForbiddenSilent("Export disabled for non-admin users."); @@ -85,10 +102,76 @@ class Service throw new ForbiddenSilent("Export disabled for '{$entityType}'."); } + if ($serviceParams->isIdle()) { + if ($this->user->isPortal()) { + throw new ForbiddenSilent("Idle export is not allowed for portal users."); + } + + return $this->schedule($params); + } + $export = $this->factory->create(); - return $export + $result = $export ->setParams($params) ->run(); + + return ServiceResult::createWithResult($result); + } + + public function getStatusData(string $id): stdClass + { + /** @var ExportEntity|null $entity */ + $entity = $this->entityManager->getEntity(ExportEntity::ENTITY_TYPE, $id); + + if (!$entity) { + throw new NotFoundSilent(); + } + + if ($entity->getCreatedBy()->getId() !== $this->user->getId()) { + throw new ForbiddenSilent(); + } + + return (object) [ + 'status' => $entity->getStatus(), + 'attachmentId' => $entity->getAttachmentId(), + ]; + } + + public function subscribeToNotificationOnSuccess(string $id): void + { + /** @var ExportEntity|null $entity */ + $entity = $this->entityManager->getEntity(ExportEntity::ENTITY_TYPE, $id); + + if (!$entity) { + throw new NotFoundSilent(); + } + + if ($entity->getCreatedBy()->getId() !== $this->user->getId()) { + throw new ForbiddenSilent(); + } + + $entity->setNotifyOnFinish(); + + $this->entityManager->saveEntity($entity); + } + + private function schedule(Params $params): ServiceResult + { + $entity = $this->entityManager->createEntity(ExportEntity::ENTITY_TYPE, [ + 'params' => serialize($params), + ]); + + $this->jobSchedulerFactory + ->create() + ->setClassName(Process::class) + ->setData( + JobData::create() + ->withTargetId($entity->getId()) + ->withTargetType($entity->getEntityType()) + ) + ->schedule(); + + return ServiceResult::createWithId($entity->getId()); } } diff --git a/application/Espo/Tools/Export/ServiceParams.php b/application/Espo/Tools/Export/ServiceParams.php new file mode 100644 index 0000000000..ccadabfb67 --- /dev/null +++ b/application/Espo/Tools/Export/ServiceParams.php @@ -0,0 +1,54 @@ +isIdle; + } + + public function withIsIdle(bool $isIdle = true): self + { + $obj = clone $this; + + $obj->isIdle = $isIdle; + + return $obj; + } +} diff --git a/application/Espo/Tools/Export/ServiceResult.php b/application/Espo/Tools/Export/ServiceResult.php new file mode 100644 index 0000000000..1da5870735 --- /dev/null +++ b/application/Espo/Tools/Export/ServiceResult.php @@ -0,0 +1,70 @@ +result !== null; + } + + public function getId(): ?string + { + return $this->id; + } + + public function getResult(): ?Result + { + return $this->result; + } + + public static function createWithId(string $id): self + { + $obj = new self; + $obj->id = $id; + + return $obj; + } + + public static function createWithResult(Result $result): self + { + $obj = new self; + $obj->result = $result; + + return $obj; + } +} diff --git a/client/res/templates/export/modals/idle.tpl b/client/res/templates/export/modals/idle.tpl new file mode 100644 index 0000000000..1ce5907b71 --- /dev/null +++ b/client/res/templates/export/modals/idle.tpl @@ -0,0 +1,7 @@ +