espocrm-extensions/espocrm-base
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

email import ref

Browse Source
This commit is contained in:
Yuri Kuznetsov
2022-11-08 14:29:12 +02:00
parent 9190544b10
commit 6e07ecd5bb
8 changed files with 469 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
application/Espo/Binding.php
+10
View File
@@ -240,6 +240,16 @@ class Binding implements BindingProcessor
'Espo\\Core\\Authentication\\Jwt\\SignatureVerifierFactory',
'Espo\\Core\\Authentication\\Oidc\\DefaultSignatureVerifierFactory'
);
$binder->bindImplementation(
'Espo\\Core\\Mail\\Importer\\ParentFinder',
'Espo\\Core\\Mail\\Importer\\DefaultParentFinder'
);
$binder->bindImplementation(
'Espo\\Core\\Mail\\Importer\\DuplicateFinder',
'Espo\\Core\\Mail\\Importer\\DefaultDuplicateFinder'
);
}
private function bindAcl(Binder $binder): void
application/Espo/Core/Mail/Importer.php
+10 -207
View File
@@ -32,10 +32,9 @@ namespace Espo\Core\Mail;
use Espo\Core\Job\Job\Data as JobData;
use Espo\Core\Job\JobSchedulerFactory;
use Espo\Core\Mail\Importer\DuplicateFinder;
use Espo\Core\Mail\Importer\ParentFinder;
use Espo\Entities\Email;
use Espo\Entities\EmailFilter;
use Espo\Modules\Crm\Entities\Account;
use Espo\Modules\Crm\Entities\Contact;
use Espo\Repositories\Email as EmailRepository;
use Espo\ORM\EntityManager;
@@ -53,8 +52,6 @@ use Espo\Core\FieldProcessing\Saver\Params as SaverParams;
use Espo\Core\Job\QueueName;
use Espo\Core\ORM\Entity as CoreEntity;
use Espo\Modules\Crm\Entities\Lead;
use DateTime;
use DateTimeZone;
use Espo\Tools\Stream\Jobs\ProcessNoteAcl;
@@ -77,6 +74,7 @@ class Importer
private LinkMultipleSaver $linkMultipleSaver;
private DuplicateFinder $duplicateFinder;
private JobSchedulerFactory $jobSchedulerFactory;
private ParentFinder $parentFinder;
public function __construct(
EntityManager $entityManager,
@@ -85,7 +83,8 @@ class Importer
ParserFactory $parserFactory,
LinkMultipleSaver $linkMultipleSaver,
DuplicateFinder $duplicateFinder,
JobSchedulerFactory $jobSchedulerFactory
JobSchedulerFactory $jobSchedulerFactory,
ParentFinder $parentFinder
) {
$this->entityManager = $entityManager;
$this->config = $config;
@@ -93,6 +92,7 @@ class Importer
$this->linkMultipleSaver = $linkMultipleSaver;
$this->duplicateFinder = $duplicateFinder;
$this->jobSchedulerFactory = $jobSchedulerFactory;
$this->parentFinder = $parentFinder;
$this->notificator = $notificatorFactory->createByClass(Email::class);
$this->filtersMatcher = new FiltersMatcher();
@@ -298,8 +298,6 @@ class Importer
$email->set('isHtml', false);
}
$replied = null;
if (
$parser->hasHeader($message, 'in-Reply-To') &&
$parser->getHeader($message, 'in-Reply-To')
@@ -313,7 +311,7 @@ class Importer
$inReplyTo = '<' . $inReplyTo . '>';
}
/** @var Email|null $replied */
/** @var ?Email $replied */
$replied = $this->entityManager
->getRDBRepository(Email::ENTITY_TYPE)
->where(['messageId' => $inReplyTo])
@@ -332,47 +330,11 @@ class Importer
}
}
$parentFound = $this->processReferences($parser, $message, $email);
$parentFound = $this->parentFinder->find($email, $message);
if (
!$parentFound &&
$replied &&
$replied->getParentId() &&
$replied->getParentType()
) {
/** @var string $parentId */
$parentId = $replied->getParentId();
/** @var string $parentType */
$parentType = $replied->getParentType();
$parentEntity = $this->entityManager->getEntityById($parentType, $parentId);
if ($parentEntity) {
$parentFound = true;
$email->set('parentType', $parentType);
$email->set('parentId', $parentId);
}
}
if (!$parentFound) {
$from = $email->getFromAddress();
if ($from) {
$parentFound = $this->findParentByAddress($email, $from);
}
}
if (!$parentFound) {
if (!empty($replyToArr)) {
$parentFound = $this->findParentByAddress($email, $replyToArr[0]);
}
}
if (!$parentFound) {
if (!empty($toArr)) {
$parentFound = $this->findParentByAddress($email, $toArr[0]);
}
if ($parentFound) {
$email->set('parentType', $parentFound->getEntityType());
$email->set('parentId', $parentFound->getId());
}
if (!$duplicate) {
@@ -509,165 +471,6 @@ class Importer
}
}
private function processReferences(Parser $parser, Message $message, Email $email): bool
{
if (
!$parser->hasHeader($message, 'references') ||
!$parser->getHeader($message, 'references')
) {
return false;
}
$references = $parser->getHeader($message, 'references');
$delimiter = strpos($references, '>,') ? ',' : ' ';
foreach (explode($delimiter, $references) as $reference) {
$reference = str_replace(['/', '@'], ' ', trim(trim($reference), '<>'));
$parentFound = $this->processReferencesItem($email, $reference);
if ($parentFound) {
return true;
}
}
return false;
}
private function processReferencesItem(Email $email, string $reference): bool
{
$parentType = null;
$parentId = null;
$number = null;
$emailSent = PHP_INT_MAX;
$n = sscanf($reference, '%s %s %d %d espo', $parentType, $parentId, $emailSent, $number);
if ($n !== 4) {
$n = sscanf($reference, '%s %s %d %d espo-system', $parentType, $parentId, $emailSent, $number);
}
if ($n !== 4 || $emailSent >= time()) {
return false;
}
if (!$parentType || !$parentId) {
return false;
}
$email->set('parentType', $parentType);
$email->set('parentId', $parentId);
if ($parentType === Lead::ENTITY_TYPE) {
/** @var ?Lead $parent */
$parent = $this->entityManager->getEntityById(Lead::ENTITY_TYPE, $parentId);
if (!$parent) {
$email->set('parentType', null);
$email->set('parentId', null);
return false;
}
$this->processReferenceLead($email, $parent);
}
return true;
}
private function processReferenceLead(Email $email, Lead $lead): void
{
if ($lead->getStatus() !== Lead::STATUS_CONVERTED) {
return;
}
$createdAccountId = $lead->get('createdAccountId');
if ($createdAccountId) {
$account = $this->entityManager->getEntityById(Account::ENTITY_TYPE, $createdAccountId);
if (!$account) {
return;
}
$email->set('parentType', Account::ENTITY_TYPE);
$email->set('parentId', $account->getId());
return;
}
$createdContactId = $lead->get('createdContactId');
if (
$this->config->get('b2cMode') &&
$createdContactId
) {
$contact = $this->entityManager->getEntityById(Contact::ENTITY_TYPE, $createdContactId);
if (!$contact) {
return;
}
$email->set('parentType', Contact::ENTITY_TYPE);
$email->set('parentId', $contact->getId());
}
}
private function findParentByAddress(Email $email, string $emailAddress): bool
{
$contact = $this->entityManager
->getRDBRepository(Contact::ENTITY_TYPE)
->where([
'emailAddress' => $emailAddress
])
->findOne();
if ($contact) {
if (!$this->config->get('b2cMode') && $contact->get('accountId')) {
$email->set('parentType', Account::ENTITY_TYPE);
$email->set('parentId', $contact->get('accountId'));
return true;
}
$email->set('parentType', Contact::ENTITY_TYPE);
$email->set('parentId', $contact->getId());
return true;
}
$account = $this->entityManager
->getRDBRepository(Account::ENTITY_TYPE)
->where([
'emailAddress' => $emailAddress
])
->findOne();
if ($account) {
$email->set('parentType', Account::ENTITY_TYPE);
$email->set('parentId', $account->getId());
return true;
}
$lead = $this->entityManager
->getRDBRepository(Lead::ENTITY_TYPE)
->where([
'emailAddress' => $emailAddress
])
->findOne();
if ($lead) {
$email->set('parentType', Lead::ENTITY_TYPE);
$email->set('parentId', $lead->getId());
return true;
}
return false;
}
private function findDuplicate(Email $email): ?Email
{
return $this->duplicateFinder->find($email);
application/Espo/Core/Mail/Importer/DefaultDuplicateFinder.php
+58
View File
@@ -0,0 +1,58 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM - Open Source CRM application.
* Copyright (C) 2014-2022 Yurii Kuznietsov, Taras Machyshyn, Oleksii Avramenko
* Website: https://www.espocrm.com
*
* EspoCRM is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* EspoCRM is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with EspoCRM. If not, see http://www.gnu.org/licenses/.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU General Public License version 3.
*
* In accordance with Section 7(b) of the GNU General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Mail\Importer;
use Espo\Entities\Email;
use Espo\ORM\EntityManager;
class DefaultDuplicateFinder implements DuplicateFinder
{
private EntityManager $entityManager;
public function __construct(EntityManager $entityManager)
{
$this->entityManager = $entityManager;
}
public function find(Email $email): ?Email
{
if (!$email->getMessageId()) {
return null;
}
return $this->entityManager
->getRDBRepositoryByClass(Email::class)
->select(['id', 'status'])
->where([
'messageId' => $email->getMessageId(),
])
->findOne();
}
}
application/Espo/Core/Mail/Importer/DefaultParentFinder.php
+310
View File
@@ -0,0 +1,310 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM - Open Source CRM application.
* Copyright (C) 2014-2022 Yurii Kuznietsov, Taras Machyshyn, Oleksii Avramenko
* Website: https://www.espocrm.com
*
* EspoCRM is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* EspoCRM is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with EspoCRM. If not, see http://www.gnu.org/licenses/.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU General Public License version 3.
*
* In accordance with Section 7(b) of the GNU General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Mail\Importer;
use Espo\Core\Mail\Message;
use Espo\Core\Templates\Entities\Company;
use Espo\Core\Templates\Entities\Person;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\Metadata;
use Espo\Entities\Email;
use Espo\Entities\EmailAddress;
use Espo\Modules\Crm\Entities\Account;
use Espo\Modules\Crm\Entities\Contact;
use Espo\Modules\Crm\Entities\Lead;
use Espo\ORM\Entity;
use Espo\ORM\EntityManager;
use Espo\Repositories\EmailAddress as EmailAddressRepository;
class DefaultParentFinder implements ParentFinder
{
/** @var string[] */
private array $entityTypeList;
private EntityManager $entityManager;
private Config $config;
private Metadata $metadata;
public function __construct(
EntityManager $entityManager,
Config $config,
Metadata $metadata
) {
$this->entityManager = $entityManager;
$this->config = $config;
$this->metadata = $metadata;
$this->entityTypeList = $this->entityManager
->getDefs()
->getEntity(Email::ENTITY_TYPE)
->getField('parent')
->getParam('entityList') ?? [];
}
public function find(Email $email, Message $message): ?Entity
{
return
$this->getByReferences($message) ??
$this->getFromReplied($email) ??
$this->getByFromAddress($email) ??
$this->getByReplyToAddress($email) ??
$this->getByToAddress($email);
}
private function isEntityTypeAllowed(string $entityType): bool
{
return in_array($entityType, $this->entityTypeList);
}
private function getByFromAddress(Email $email): ?Entity
{
$from = $email->getFromAddress();
if (!$from) {
return null;
}
return $this->getByAddress($from);
}
private function getByReplyToAddress(Email $email): ?Entity
{
$list = $email->getReplyToAddressList();
if ($list === []) {
return null;
}
return $this->getByAddress($list[0]);
}
private function getByToAddress(Email $email): ?Entity
{
$list = $email->getToAddressList();
if ($list === []) {
return null;
}
return $this->getByAddress($list[0]);
}
private function getByAddress(string $emailAddress): ?Entity
{
/** @var ?Contact $contact */
$contact = $this->entityManager
->getRDBRepository(Contact::ENTITY_TYPE)
->where([
'emailAddress' => $emailAddress
])
->findOne();
if ($contact) {
$accountLink = $contact->getAccount();
if (
!$this->config->get('b2cMode') &&
$accountLink &&
$this->isEntityTypeAllowed(Account::ENTITY_TYPE)
) {
return $this->entityManager->getEntityById(Account::ENTITY_TYPE, $accountLink->getId());
}
if ($this->isEntityTypeAllowed(Contact::ENTITY_TYPE)) {
return $contact;
}
}
if ($this->isEntityTypeAllowed(Account::ENTITY_TYPE)) {
$account = $this->entityManager
->getRDBRepository(Account::ENTITY_TYPE)
->where([
'emailAddress' => $emailAddress
])
->findOne();
if ($account) {
return $account;
}
}
if ($this->isEntityTypeAllowed(Lead::ENTITY_TYPE)) {
$lead = $this->entityManager
->getRDBRepository(Lead::ENTITY_TYPE)
->where([
'emailAddress' => $emailAddress
])
->findOne();
if ($lead) {
return $lead;
}
}
$entityTypeList = array_filter(
$this->entityTypeList,
function ($entityType) {
return
!in_array(
$entityType,
[Account::ENTITY_TYPE, Contact::ENTITY_TYPE, Lead::ENTITY_TYPE]
) &&
in_array(
$this->metadata->get(['scopes', $entityType, 'type']),
[Company::TEMPLATE_TYPE, Person::TEMPLATE_TYPE]
);
}
);
/** @var EmailAddressRepository $emailAddressRepository */
$emailAddressRepository = $this->entityManager->getRepository(EmailAddress::ENTITY_TYPE);
foreach ($entityTypeList as $entityType) {
$entity = $emailAddressRepository->getEntityByAddress($emailAddress, $entityType);
if ($entity) {
return $entity;
}
}
return null;
}
private function getFromReplied(Email $email): ?Entity
{
$repliedLink = $email->getReplied();
if (!$repliedLink) {
return null;
}
/** @var ?Email $repliedEmail */
$repliedEmail = $this->entityManager
->getRDBRepositoryByClass(Email::class)
->getById($repliedLink->getId());
if (!$repliedEmail) {
return null;
}
$parentLink = $repliedEmail->getParent();
if (!$parentLink) {
return null;
}
if (!$this->entityManager->hasRepository($parentLink->getEntityType())) {
return null;
}
return $this->entityManager->getEntityById($parentLink->getEntityType(), $parentLink->getId());
}
private function getByReferences(Message $message): ?Entity
{
$references = $message->getHeader('references');
if (!$references) {
return null;
}
$delimiter = strpos($references, '>,') ? ',' : ' ';
foreach (explode($delimiter, $references) as $reference) {
$reference = str_replace(['/', '@'], ' ', trim(trim($reference), '<>'));
$parent = $this->getByReferencesItem($reference);
if ($parent) {
return $parent;
}
}
return null;
}
private function getByReferencesItem(string $reference): ?Entity
{
$parentType = null;
$parentId = null;
$number = null;
$emailSent = PHP_INT_MAX;
$n = sscanf($reference, '%s %s %d %d espo', $parentType, $parentId, $emailSent, $number);
if ($n !== 4) {
$n = sscanf($reference, '%s %s %d %d espo-system', $parentType, $parentId, $emailSent, $number);
}
if ($n !== 4 || $emailSent >= time()) {
return null;
}
if (!$parentType || !$parentId) {
return null;
}
if (!$this->entityManager->hasRepository($parentType)) {
return null;
}
$parent = $this->entityManager->getEntityById($parentType, $parentId);
if ($parent instanceof Lead) {
return $this->getFromLead($parent) ?? $parent;
}
return $parent;
}
private function getFromLead(Lead $lead): ?Entity
{
if ($lead->getStatus() !== Lead::STATUS_CONVERTED) {
return null;
}
$createdAccountLink = $lead->getCreatedAccount();
if ($createdAccountLink) {
return $this->entityManager->getEntityById(Account::ENTITY_TYPE, $createdAccountLink->getId());
}
$createdContactLink = $lead->getCreatedContact();
if (
$this->config->get('b2cMode') &&
$createdContactLink
) {
return $this->entityManager->getEntityById(Contact::ENTITY_TYPE, $createdContactLink->getId());
}
return null;
}
}
application/Espo/Core/Mail/Importer/DuplicateFinder.php
+5 -23
View File
@@ -30,29 +30,11 @@
namespace Espo\Core\Mail\Importer;
use Espo\Entities\Email;
use Espo\ORM\EntityManager;
class DuplicateFinder
/**
* Finds an existing duplicate of an email being imported.
*/
interface DuplicateFinder
{
private EntityManager $entityManager;
public function __construct(EntityManager $entityManager)
{
$this->entityManager = $entityManager;
}
public function find(Email $email): ?Email
{
if (!$email->getMessageId()) {
return null;
}
return $this->entityManager
->getRDBRepositoryByClass(Email::class)
->select(['id', 'status'])
->where([
'messageId' => $email->getMessageId(),
])
->findOne();
}
public function find(Email $email): ?Email;
}
application/Espo/Core/Mail/Importer/ParentFinder.php
+42
View File
@@ -0,0 +1,42 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM - Open Source CRM application.
* Copyright (C) 2014-2022 Yurii Kuznietsov, Taras Machyshyn, Oleksii Avramenko
* Website: https://www.espocrm.com
*
* EspoCRM is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* EspoCRM is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with EspoCRM. If not, see http://www.gnu.org/licenses/.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU General Public License version 3.
*
* In accordance with Section 7(b) of the GNU General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Mail\Importer;
use Espo\Entities\Email;
use Espo\Core\Mail\Message;
use Espo\ORM\Entity;
/**
* Finds a parent record for an email being imported.
*/
interface ParentFinder
{
public function find(Email $email, Message $message): ?Entity;
}
application/Espo/Modules/Crm/Entities/Lead.php
+18
View File
@@ -111,4 +111,22 @@ class Lead extends \Espo\Core\Entities\Person
/** @var LinkMultiple */
return $this->getValueObject('teams');
}
public function getCreatedAccount(): ?Link
{
/** @var ?Link */
return $this->getValueObject('createdAccount');
}
public function getCreatedContact(): ?Link
{
/** @var ?Link */
return $this->getValueObject('createdContact');
}
public function getCreatedOpportunity(): ?Link
{
/** @var ?Link */
return $this->getValueObject('createdOpportunity');
}
}
tests/unit/Espo/Core/Mail/ImporterTest.php
+16 -15
View File
@@ -36,20 +36,18 @@ use Espo\Core\Notification\AssignmentNotificator;
use Espo\ORM\Value\ValueAccessor;
use Espo\ORM\Value\ValueAccessorFactory;
use Espo\Core\{
Job\JobSchedulerFactory,
Mail\Importer,
Mail\Importer\Data as ImporterData,
Mail\MessageWrapper,
Mail\ParserFactory,
Mail\Parsers\MailMimeParser,
Utils\Log,
ORM\EntityManager,
Utils\Config,
Repositories\Database,
Utils\Metadata,
Notification\AssignmentNotificatorFactory,
FieldProcessing\Relation\LinkMultipleSaver};
use Espo\Core\FieldProcessing\Relation\LinkMultipleSaver;
use Espo\Core\Job\JobSchedulerFactory;
use Espo\Core\Mail\Importer;
use Espo\Core\Mail\Importer\Data as ImporterData;
use Espo\Core\Mail\MessageWrapper;
use Espo\Core\Mail\ParserFactory;
use Espo\Core\Mail\Parsers\MailMimeParser;
use Espo\Core\Notification\AssignmentNotificatorFactory;
use Espo\Core\ORM\EntityManager;
use Espo\Core\Repositories\Database;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\Metadata;
use Espo\ORM\Repository\RDBSelectBuilder;
@@ -72,6 +70,8 @@ class ImporterTest extends \PHPUnit\Framework\TestCase
$this->parserFactory = $this->createMock(ParserFactory::class);
$this->linkMultipleSaver = $this->createMock(LinkMultipleSaver::class);
$this->parentFinder = $this->createMock(Importer\ParentFinder::class);
$this->parserFactory
->expects($this->any())
->method('create')
@@ -186,7 +186,8 @@ class ImporterTest extends \PHPUnit\Framework\TestCase
$this->parserFactory,
$this->linkMultipleSaver,
$this->duplicateFinder,
$this->jobSchedulerFactory
$this->jobSchedulerFactory,
$this->parentFinder
);
$message = new MessageWrapper(0, null, null, $contents);