diff --git a/application/Espo/Core/Mail/Importer/DefaultImporter.php b/application/Espo/Core/Mail/Importer/DefaultImporter.php index 1825c34551..d648be5b7a 100644 --- a/application/Espo/Core/Mail/Importer/DefaultImporter.php +++ b/application/Espo/Core/Mail/Importer/DefaultImporter.php @@ -38,9 +38,9 @@ use Espo\Core\Mail\FiltersMatcher; use Espo\Core\Mail\Importer; use Espo\Core\Mail\Message; use Espo\Core\Mail\MessageWrapper; +use Espo\Core\Mail\Parser; use Espo\Core\Mail\ParserFactory; use Espo\Core\ORM\Repository\Option\SaveOption; -use Espo\Core\Utils\DateTime as DateTimeUtil; use Espo\Core\Notification\AssignmentNotificator; use Espo\Core\Notification\AssignmentNotificatorFactory; use Espo\Core\Notification\AssignmentNotificator\Params as AssignmentNotificatorParams; @@ -49,6 +49,7 @@ use Espo\Core\FieldProcessing\Relation\LinkMultipleSaver; use Espo\Core\FieldProcessing\Saver\Params as SaverParams; use Espo\Core\Job\QueueName; use Espo\Core\ORM\Entity as CoreEntity; +use Espo\Entities\Attachment; use Espo\Entities\Email; use Espo\Entities\EmailFilter; use Espo\Repositories\Email as EmailRepository; @@ -56,7 +57,6 @@ use Espo\ORM\EntityManager; use Espo\Tools\Stream\Jobs\ProcessNoteAcl; use DateTime; -use DateTimeZone; use Exception; class DefaultImporter implements Importer @@ -84,226 +84,87 @@ class DefaultImporter implements Importer public function import(Message $message, Data $data): ?Email { - $assignedUserId = $data->getAssignedUserId(); - $teamIdList = $data->getTeamIdList(); - $userIdList = $data->getUserIdList(); - $filterList = $data->getFilterList(); - $folderData = $data->getFolderData(); - $groupEmailFolderId = $data->getGroupEmailFolderId(); - - $parser = $message instanceof MessageWrapper ? - ($message->getParser() ?? $this->parserFactory->create()) : - $this->parserFactory->create(); - - /** @var Email $email */ - $email = $this->entityManager->getNewEntity(Email::ENTITY_TYPE); + $parser = $this->getParser($message); + $email = $this->entityManager->getRDBRepositoryByClass(Email::class)->getNew(); $email->set('isBeingImported', true); - $subject = ''; + $subject = $this->getSubject($parser, $message); - if ($parser->hasHeader($message, 'subject')) { - $subject = $parser->getHeader($message, 'subject'); + $email + ->setSubject($subject) + ->setStatus(Email::STATUS_ARCHIVED) + ->setIsHtml(false) + ->setGroupFolderId($data->getGroupEmailFolderId()) + ->setTeams(LinkMultiple::create()->withAddedIdList($data->getTeamIdList())); + + if ($data->getAssignedUserId()) { + $email->setAssignedUserId($data->getAssignedUserId()); + $email->addAssignedUserId($data->getAssignedUserId()); } - if (!empty($subject) && is_string($subject)) { - $subject = trim($subject); - } - - if ($subject !== '0' && empty($subject)) { - $subject = '(No Subject)'; - } - - if (strlen($subject) > self::SUBJECT_MAX_LENGTH) { - $subject = substr($subject, 0, self::SUBJECT_MAX_LENGTH); - } - - $email->setSubject($subject); - $email->setStatus(Email::STATUS_ARCHIVED); - $email->setIsHtml(false); - $email->setGroupFolderId($groupEmailFolderId); - $email->setTeams(LinkMultiple::create()->withAddedIdList($teamIdList)); - //$email->set('attachmentsIds', []); - - if ($assignedUserId) { - $email->setAssignedUserId($assignedUserId); - $email->addAssignedUserId($assignedUserId); - } - - foreach ($userIdList as $uId) { + foreach ($data->getUserIdList() as $uId) { $email->addUserId($uId); } - $fromAddressData = $parser->getAddressData($message, 'from'); + $this->setFromStrings($parser, $message, $email); + $this->setAddresses($parser, $message, $email); - if ($fromAddressData) { - $fromString = ($fromAddressData->name ? ($fromAddressData->name . ' ') : '') . '<' . - $fromAddressData->address . '>'; - - $email->set('fromString', $fromString); - } - - $replyToData = $parser->getAddressData($message, 'reply-To'); - - if ($replyToData) { - $replyToString = ($replyToData->name ? ($replyToData->name . ' ') : '') . - '<' . $replyToData->address . '>'; - - $email->set('replyToString', $replyToString); - } - - $fromArr = $parser->getAddressList($message, 'from'); - $toArr = $parser->getAddressList($message, 'to'); - $ccArr = $parser->getAddressList($message, 'cc'); - $replyToArr = $parser->getAddressList($message, 'reply-To'); - - $email->setFromAddress($fromArr[0] ?? null); - $email->setToAddressList($toArr); - $email->setCcAddressList($ccArr); - $email->setReplyToAddressList($replyToArr); - - $addressNameMap = $parser->getAddressNameMap($message); - - $email->set('addressNameMap', $addressNameMap); - - foreach ($folderData as $uId => $folderId) { + foreach ($data->getFolderData() as $uId => $folderId) { $email->setUserColumnFolderId($uId, $folderId); } - $matchedFilter = $this->filtersMatcher->findMatch($email, $filterList, true); + $toSkip = $this->processFilters($email, $data->getFilterList(), true); - if ($matchedFilter && $matchedFilter->getAction() === EmailFilter::ACTION_SKIP) { + if ($toSkip) { return null; } - if ($matchedFilter && $matchedFilter->getAction() === EmailFilter::ACTION_MOVE_TO_GROUP_FOLDER) { - $groupEmailFolderId = $matchedFilter->getGroupEmailFolderId(); + $isSystemEmail = $this->processMessageId($parser, $message, $email); - $email->setGroupFolderId($groupEmailFolderId); + if ($isSystemEmail) { + return null; } - if ( - $parser->hasHeader($message, 'message-Id') && - $parser->getHeader($message, 'message-Id') - ) { - /** @var string $messageId */ - $messageId = $parser->getMessageId($message); - - $email->setMessageId($messageId); - - if ($parser->hasHeader($message, 'delivered-To')) { - $messageIdInternal = $messageId . '-' . $parser->getHeader($message, 'delivered-To'); - - $email->set('messageIdInternal', $messageIdInternal); - } - - if (stripos($messageId, '@espo-system') !== false) { - return null; - } - } - - if ($parser->hasHeader($message, 'date')) { - try { - $dateHeaderValue = $parser->getHeader($message, 'date') ?? 'now'; - - $dateSent = new DateTime($dateHeaderValue); - - $email->setDateSent(DateTimeField::fromDateTime($dateSent)); - } catch (Exception) {} - } + $this->processDate($parser, $message, $email); $duplicate = $this->findDuplicate($email, $message); if ($duplicate && $duplicate->getStatus() !== Email::STATUS_BEING_IMPORTED) { $this->entityManager->refreshEntity($duplicate); - $this->processDuplicate( - $duplicate, - $assignedUserId, - $userIdList, - $folderData, - $teamIdList, - $groupEmailFolderId - ); + $this->processDuplicate($duplicate, $data, $email->getGroupFolder()?->getId()); return $duplicate; } + $this->processDeliveryDate($parser, $message, $email); + if (!$email->getDateSent()) { $email->setDateSent(DateTimeField::createNow()); } - if ($parser->hasHeader($message, 'delivery-Date')) { - try { - /** @var string $deliveryDateHeaderValue */ - $deliveryDateHeaderValue = $parser->getHeader($message, 'delivery-Date'); - - $dt = new DateTime($deliveryDateHeaderValue); - - $deliveryDate = $dt - ->setTimezone(new DateTimeZone('UTC')) - ->format(DateTimeUtil::SYSTEM_DATE_TIME_FORMAT); - - $email->set('deliveryDate', $deliveryDate); - } catch (Exception) {} - } - $inlineAttachmentList = []; if (!$data->fetchOnlyHeader()) { $inlineAttachmentList = $parser->getInlineAttachmentList($message, $email); - $matchedFilter = $this->filtersMatcher->findMatch($email, $filterList); + $toSkip = $this->processFilters($email, $data->getFilterList()); - if ($matchedFilter && $matchedFilter->getAction() === EmailFilter::ACTION_SKIP) { + if ($toSkip) { return null; } - - if ($matchedFilter && $matchedFilter->getAction() === EmailFilter::ACTION_MOVE_TO_GROUP_FOLDER) { - $groupEmailFolderId = $matchedFilter->getGroupEmailFolderId(); - - $email->setGroupFolderId($groupEmailFolderId); - } } else { $email->setBody('Not fetched. The email size exceeds the limit.'); $email->setIsHtml(false); } - if ( - $parser->hasHeader($message, 'in-Reply-To') && - $parser->getHeader($message, 'in-Reply-To') - ) { - $arr = explode(' ', $parser->getHeader($message, 'in-Reply-To')); - - $inReplyTo = $arr[0]; - - if ($inReplyTo) { - if ($inReplyTo[0] !== '<') { - $inReplyTo = '<' . $inReplyTo . '>'; - } - - /** @var ?Email $replied */ - $replied = $this->entityManager - ->getRDBRepository(Email::ENTITY_TYPE) - ->where(['messageId' => $inReplyTo]) - ->findOne(); - - if ($replied) { - $email->setRepliedId($replied->getId()); - - $repliedTeamIdList = $replied->getLinkMultipleIdList('teams'); - - foreach ($repliedTeamIdList as $repliedTeamId) { - $email->addTeamId($repliedTeamId); - } - } - } - } + $this->processInReplyTo($parser, $message, $email); $parentFound = $this->parentFinder->find($email, $message); if ($parentFound) { - $email->setParent(LinkParent::create($parentFound->getEntityType(), $parentFound->getId())); + $email->setParent(LinkParent::createFromEntity($parentFound)); } if (!$duplicate) { @@ -317,14 +178,7 @@ class DefaultImporter implements Importer if ($duplicate->getStatus() !== Email::STATUS_BEING_IMPORTED) { $this->entityManager->refreshEntity($duplicate); - $this->processDuplicate( - $duplicate, - $assignedUserId, - $userIdList, - $folderData, - $teamIdList, - $groupEmailFolderId - ); + $this->processDuplicate($duplicate, $data, $email->getGroupFolder()?->getId()); return $duplicate; } @@ -333,20 +187,9 @@ class DefaultImporter implements Importer if ($duplicate) { $this->copyAttributesToDuplicate($email, $duplicate); + $this->getEmailRepository()->fillAccount($duplicate); - /** @var EmailRepository $emailRepository */ - $emailRepository = $this->entityManager->getRDBRepository(Email::ENTITY_TYPE); - - $emailRepository->fillAccount($duplicate); - - $this->processDuplicate( - $duplicate, - $assignedUserId, - $userIdList, - $folderData, - $teamIdList, - $groupEmailFolderId - ); + $this->processDuplicate($duplicate, $data, $email->getGroupFolder()?->getId()); return $duplicate; } @@ -370,23 +213,8 @@ class DefaultImporter implements Importer $email->setStatus(Email::STATUS_ARCHIVED); - $this->entityManager->getTransactionManager()->start(); - $this->entityManager - ->getRDBRepositoryByClass(Email::class) - ->forUpdate() - ->where(['id' => $email->getId()]) - ->findOne(); - - $this->entityManager->saveEntity($email, [Email::SAVE_OPTION_IS_BEING_IMPORTED => true]); - - $this->entityManager->getTransactionManager()->commit(); - - foreach ($inlineAttachmentList as $attachment) { - $attachment->setTargetField('body'); - $attachment->setRelated(LinkParent::create(Email::ENTITY_TYPE, $email->getId())); - - $this->entityManager->saveEntity($attachment); - } + $this->processFinalTransactionalSave($email); + $this->processAttachmentSave($inlineAttachmentList, $email); return $email; } @@ -452,26 +280,13 @@ class DefaultImporter implements Importer return $this->duplicateFinder->find($email, $message); } - /** - * @param string[] $userIdList - * @param array $folderData - * @param string[] $teamIdList - */ - private function processDuplicate( - Email $duplicate, - ?string $assignedUserId, - array $userIdList, - array $folderData, - array $teamIdList, - ?string $groupEmailFolderId - ): void { - - /** @var EmailRepository $emailRepository */ - $emailRepository = $this->entityManager->getRDBRepository(Email::ENTITY_TYPE); + private function processDuplicate(Email $duplicate, Data $data, ?string $groupEmailFolderId): void + { + $assignedUserId = $data->getAssignedUserId(); if ($duplicate->getStatus() === Email::STATUS_ARCHIVED) { - $emailRepository->loadFromField($duplicate); - $emailRepository->loadToField($duplicate); + $this->getEmailRepository()->loadFromField($duplicate); + $this->getEmailRepository()->loadToField($duplicate); } $duplicate->loadLinkMultipleField('users'); @@ -491,7 +306,7 @@ class DefaultImporter implements Importer $duplicate->addAssignedUserId($assignedUserId); } - foreach ($userIdList as $uId) { + foreach ($data->getUserIdList() as $uId) { if (!in_array($uId, $fetchedUserIdList)) { $processNoteAcl = true; @@ -499,7 +314,7 @@ class DefaultImporter implements Importer } } - foreach ($folderData as $uId => $folderId) { + foreach ($data->getFolderData() as $uId => $folderId) { if (!in_array($uId, $fetchedUserIdList)) { $duplicate->setUserColumnFolderId($uId, $folderId); @@ -507,7 +322,6 @@ class DefaultImporter implements Importer } $this->entityManager - ->getRDBRepository(Email::ENTITY_TYPE) ->getRelation($duplicate, 'users') // Can cause skip-notification bypass. ->updateColumnsById($uId, [Email::USERS_COLUMN_FOLDER_ID => $folderId]); @@ -515,7 +329,7 @@ class DefaultImporter implements Importer $duplicate->set('isBeingImported', true); - $emailRepository->applyUsersFilters($duplicate); + $this->getEmailRepository()->applyUsersFilters($duplicate); $saverParams = SaverParams::create()->withRawOptions([ 'skipLinkMultipleRemove' => true, @@ -526,21 +340,19 @@ class DefaultImporter implements Importer $this->linkMultipleSaver->process($duplicate, 'assignedUsers', $saverParams); if ($this->emailNotificationsEnabled()) { - $this->notificator->process( - $duplicate, - AssignmentNotificatorParams::create() - ->withRawOptions([Email::SAVE_OPTION_IS_BEING_IMPORTED => true]) - ); + $notificatorParams = AssignmentNotificatorParams::create() + ->withRawOptions([Email::SAVE_OPTION_IS_BEING_IMPORTED => true]); + + $this->notificator->process($duplicate, $notificatorParams); } $fetchedTeamIdList = $duplicate->getLinkMultipleIdList('teams'); - foreach ($teamIdList as $teamId) { + foreach ($data->getTeamIdList() as $teamId) { if (!in_array($teamId, $fetchedTeamIdList)) { $processNoteAcl = true; $this->entityManager - ->getRDBRepository(Email::ENTITY_TYPE) ->getRelation($duplicate, 'teams') ->relateById($teamId); } @@ -548,7 +360,6 @@ class DefaultImporter implements Importer if ($groupEmailFolderId && !$duplicate->getGroupFolder()) { $this->entityManager - ->getRDBRepository(Email::ENTITY_TYPE) ->getRelation($duplicate, 'groupFolder') ->relateById($groupEmailFolderId); } @@ -582,4 +393,229 @@ class DefaultImporter implements Importer $this->config->get('assignmentNotificationsEntityList') ?? [] ); } + + private function getSubject(Parser $parser, Message $message): string + { + $subject = ''; + + if ($parser->hasHeader($message, 'subject')) { + $subject = $parser->getHeader($message, 'subject'); + } + + if (!empty($subject) && is_string($subject)) { + $subject = trim($subject); + } + + if ($subject !== '0' && empty($subject)) { + $subject = '(No Subject)'; + } + + if (strlen($subject) > self::SUBJECT_MAX_LENGTH) { + $subject = substr($subject, 0, self::SUBJECT_MAX_LENGTH); + } + + return $subject; + } + + private function setFromStrings(Parser $parser, Message $message, Email $email): void + { + $fromAddressData = $parser->getAddressData($message, 'from'); + + if ($fromAddressData) { + $namePart = ($fromAddressData->name ? ($fromAddressData->name . ' ') : ''); + + $email->set('fromString', "$namePart<$fromAddressData->address>"); + } + + $replyToData = $parser->getAddressData($message, 'reply-To'); + + if ($replyToData) { + $namePart = ($replyToData->name ? ($replyToData->name . ' ') : ''); + + $email->set('replyToString', "$namePart<$replyToData->address>"); + } + } + + private function setAddresses(Parser $parser, Message $message, Email $email): void + { + $from = $parser->getAddressList($message, 'from'); + $to = $parser->getAddressList($message, 'to'); + $cc = $parser->getAddressList($message, 'cc'); + $replyTo = $parser->getAddressList($message, 'reply-To'); + + $email->setFromAddress($from[0] ?? null); + $email->setToAddressList($to); + $email->setCcAddressList($cc); + $email->setReplyToAddressList($replyTo); + + $email->set('addressNameMap', $parser->getAddressNameMap($message)); + } + + /** + * @return bool True if an email is system. + */ + private function processMessageId(Parser $parser, Message $message, Email $email): bool + { + if (!$parser->hasHeader($message, 'message-Id')) { + return false; + } + + $messageId = $parser->getMessageId($message); + + if (!$messageId) { + return false; + } + + $email->setMessageId($messageId); + + if ($parser->hasHeader($message, 'delivered-To')) { + $deliveredTo = $parser->getHeader($message, 'delivered-To') ?? ''; + + $email->set('messageIdInternal', "$messageId-$deliveredTo"); + } + + if (stripos($messageId, '@espo-system') !== false) { + return true; + } + + return false; + } + + private function processDate(Parser $parser, Message $message, Email $email): void + { + if (!$parser->hasHeader($message, 'date')) { + return; + } + + $dateString = $parser->getHeader($message, 'date') ?? 'now'; + + try { + $dateSent = DateTimeField::fromDateTime(new DateTime($dateString)); + } catch (Exception) { + return; + } + + $email->setDateSent($dateSent); + } + + private function processDeliveryDate(Parser $parser, Message $message, Email $email): void + { + if (!$parser->hasHeader($message, 'delivery-Date')) { + return; + } + + $dateString = $parser->getHeader($message, 'delivery-Date') ?? 'now'; + + try { + $deliveryDate = DateTimeField::fromDateTime(new DateTime($dateString)); + } catch (Exception) { + return; + } + + $email->setDeliveryDate($deliveryDate); + } + + private function processInReplyTo(Parser $parser, Message $message, Email $email): void + { + if (!$parser->hasHeader($message, 'in-Reply-To')) { + return; + } + + $stringValue = $parser->getHeader($message, 'in-Reply-To'); + + if (!$stringValue) { + return; + } + + $values = explode(' ', $stringValue); + + $inReplyTo = $values[0] ?? null; + + if (!$inReplyTo) { + return; + } + + if ($inReplyTo[0] !== '<') { + $inReplyTo = "<$inReplyTo>"; + } + + $replied = $this->entityManager + ->getRDBRepositoryByClass(Email::class) + ->where(['messageId' => $inReplyTo]) + ->findOne(); + + if (!$replied) { + return; + } + + $email->setRepliedId($replied->getId()); + + foreach ($replied->getTeams()->getIdList() as $teamId) { + $email->addTeamId($teamId); + } + } + + /** + * @param iterable $filterList + * @return bool True if to skip. + */ + private function processFilters(Email $email, iterable $filterList, bool $skipBody = false): bool + { + $matchedFilter = $this->filtersMatcher->findMatch($email, $filterList, $skipBody); + + if (!$matchedFilter) { + return false; + } + + if ($matchedFilter->getAction() === EmailFilter::ACTION_SKIP) { + return true; + } + + if ($matchedFilter->getAction() === EmailFilter::ACTION_MOVE_TO_GROUP_FOLDER) { + $email->setGroupFolderId($matchedFilter->getGroupEmailFolderId()); + } + + return false; + } + + private function processFinalTransactionalSave(Email $email): void + { + $this->entityManager->getTransactionManager()->start(); + + $this->entityManager + ->getRDBRepositoryByClass(Email::class) + ->forUpdate() + ->where(['id' => $email->getId()]) + ->findOne(); + + $this->entityManager->saveEntity($email, [Email::SAVE_OPTION_IS_BEING_IMPORTED => true]); + + $this->entityManager->getTransactionManager()->commit(); + } + + /** + * @param Attachment[] $inlineAttachmentList + */ + private function processAttachmentSave(array $inlineAttachmentList, Email $email): void + { + foreach ($inlineAttachmentList as $attachment) { + $attachment->setTargetField('body'); + $attachment->setRelated(LinkParent::createFromEntity($email)); + + $this->entityManager->saveEntity($attachment); + } + } + + private function getParser(Message $message): Parser + { + return $message instanceof MessageWrapper ? + ($message->getParser() ?? $this->parserFactory->create()) : + $this->parserFactory->create(); + } + + private function getEmailRepository(): EmailRepository + { + /** @var EmailRepository */ + return $this->entityManager->getRDBRepository(Email::ENTITY_TYPE); + } } diff --git a/application/Espo/Entities/Email.php b/application/Espo/Entities/Email.php index c347d5245f..e4d25ea5a5 100644 --- a/application/Espo/Entities/Email.php +++ b/application/Espo/Entities/Email.php @@ -358,6 +358,12 @@ class Email extends Entity return $this->getValueObject('dateSent'); } + public function getDeliveryDate(): ?DateTime + { + /** @var ?DateTime */ + return $this->getValueObject('deliveryDate'); + } + public function getSubject(): ?string { return $this->get('subject'); @@ -791,6 +797,13 @@ class Email extends Entity return $this; } + public function setDeliveryDate(?DateTime $deliveryDate): self + { + $this->setValueObject('deliveryDate', $deliveryDate); + + return $this; + } + public function setAssignedUserId(?string $assignedUserId): self { $this->set('assignedUserId', $assignedUserId);