From f0413b7037aa7717de54570ea0e2703e59427c56 Mon Sep 17 00:00:00 2001 From: Yuri Kuznetsov Date: Thu, 5 May 2022 17:39:36 +0300 Subject: [PATCH] message partList and bounced recognizer --- .../GroupAccount/BouncedRecognizer.php | 111 ++++++++++++++++++ .../GroupAccount/Hooks/BeforeFetch.php | 45 ++----- application/Espo/Core/Mail/Message.php | 7 ++ .../Core/Mail/Message/MailMimeParser/Part.php | 79 +++++++++++++ application/Espo/Core/Mail/Message/Part.php | 47 ++++++++ application/Espo/Core/Mail/MessageWrapper.php | 24 ++++ application/Espo/Core/Mail/Parser.php | 6 + .../Espo/Core/Mail/Parsers/MailMimeParser.php | 20 ++++ .../Mail/Account/BouncedRecognizerTest.php | 96 +++++++++++++++ tests/unit/testData/Core/Mail/bounced_1.eml | 65 ++++++++++ tests/unit/testData/Core/Mail/bounced_2.eml | 41 +++++++ 11 files changed, 509 insertions(+), 32 deletions(-) create mode 100644 application/Espo/Core/Mail/Account/GroupAccount/BouncedRecognizer.php create mode 100644 application/Espo/Core/Mail/Message/MailMimeParser/Part.php create mode 100644 application/Espo/Core/Mail/Message/Part.php create mode 100644 tests/unit/Espo/Core/Mail/Account/BouncedRecognizerTest.php create mode 100644 tests/unit/testData/Core/Mail/bounced_1.eml create mode 100644 tests/unit/testData/Core/Mail/bounced_2.eml diff --git a/application/Espo/Core/Mail/Account/GroupAccount/BouncedRecognizer.php b/application/Espo/Core/Mail/Account/GroupAccount/BouncedRecognizer.php new file mode 100644 index 0000000000..14f7a280b3 --- /dev/null +++ b/application/Espo/Core/Mail/Account/GroupAccount/BouncedRecognizer.php @@ -0,0 +1,111 @@ +getHeader('From'); + $contentType = $message->getHeader('Content-Type'); + + if (preg_match('/MAILER-DAEMON|POSTMASTER/i', $from ?? '')) { + return true; + } + + if (strpos($contentType ?? '', 'multipart/report') === 0) { + // @todo Check whether ever works. + $deliveryStatusPart = $this->getDeliveryStatusPart($message); + + if ($deliveryStatusPart) { + return true; + } + + $content = $message->getRawContent(); + + if ( + strpos($content, 'message/delivery-status') !== false && + strpos($content, 'Status: ') !== false + ) { + return true; + } + } + + return false; + } + + public function isHard(Message $message): bool + { + $content = $message->getRawContent(); + + if (preg_match('/permanent[ ]*[error|failure]/', $content)) { + return true; + } + + return false; + } + + public function extractQueueItemId(Message $message): ?string + { + $content = $message->getRawContent(); + + if (preg_match('/X-Queue-Item-Id: [a-z0-9\-]*/', $content, $m)) { + /** @var array{string} */ + $arr = preg_split('/X-Queue-Item-Id: /', $m[0], -1, \PREG_SPLIT_NO_EMPTY); + + return $arr[0]; + } + + $to = $message->getHeader('to'); + + if (preg_match('/\+bounce-qid-[a-z0-9\-]*/', $to ?? '', $m)) { + /** @var array{string} */ + $arr = preg_split('/\+bounce-qid-/', $m[0], -1, \PREG_SPLIT_NO_EMPTY); + + return $arr[0]; + } + + return null; + } + + private function getDeliveryStatusPart(Message $message): ?Part + { + foreach ($message->getPartList() as $part) { + if ($part->getContentType() === 'message/delivery-status') { + return $part; + } + } + + return null; + } +} diff --git a/application/Espo/Core/Mail/Account/GroupAccount/Hooks/BeforeFetch.php b/application/Espo/Core/Mail/Account/GroupAccount/Hooks/BeforeFetch.php index 3217530129..d1e87c3ef8 100644 --- a/application/Espo/Core/Mail/Account/GroupAccount/Hooks/BeforeFetch.php +++ b/application/Espo/Core/Mail/Account/GroupAccount/Hooks/BeforeFetch.php @@ -34,6 +34,7 @@ use Espo\Core\Mail\Account\Hook\BeforeFetchResult; use Espo\Core\Mail\Account\Account; use Espo\Core\Mail\Message; +use Espo\Core\Mail\Account\GroupAccount\BouncedRecognizer; use Espo\Core\Utils\Log; use Espo\Core\InjectableFactory; @@ -55,21 +56,25 @@ class BeforeFetch implements BeforeFetchInterface private InjectableFactory $injectableFactory; + private BouncedRecognizer $bouncedRecognizer; + private ?CampaignService $campaignService = null; - public function __construct(Log $log, EntityManager $entityManager, InjectableFactory $injectableFactory) - { + public function __construct( + Log $log, + EntityManager $entityManager, + InjectableFactory $injectableFactory, + BouncedRecognizer $bouncedRecognizer + ) { $this->log = $log; $this->entityManager = $entityManager; $this->injectableFactory = $injectableFactory; + $this->bouncedRecognizer = $bouncedRecognizer; } public function process(Account $account, Message $message): BeforeFetchResult { - if ( - $message->hasHeader('from') && - preg_match('/MAILER-DAEMON|POSTMASTER/i', $message->getHeader('from') ?? '') - ) { + if ($this->bouncedRecognizer->isBounced($message)) { try { $toSkip = $this->processBounced($message); } @@ -94,32 +99,8 @@ class BeforeFetch implements BeforeFetchInterface private function processBounced(Message $message): bool { - $content = $message->getRawContent(); - - $isHard = false; - - if (preg_match('/permanent[ ]*[error|failure]/', $content)) { - $isHard = true; - } - - $queueItemId = null; - - if (preg_match('/X-Queue-Item-Id: [a-z0-9\-]*/', $content, $m)) { - /** @var array{string} */ - $arr = preg_split('/X-Queue-Item-Id: /', $m[0], -1, \PREG_SPLIT_NO_EMPTY); - - $queueItemId = $arr[0]; - } - else { - $to = $message->getHeader('to'); - - if (preg_match('/\+bounce-qid-[a-z0-9\-]*/', $to ?? '', $m)) { - /** @var array{string} */ - $arr = preg_split('/\+bounce-qid-/', $m[0], -1, \PREG_SPLIT_NO_EMPTY); - - $queueItemId = $arr[0]; - } - } + $isHard = $this->bouncedRecognizer->isHard($message); + $queueItemId = $this->bouncedRecognizer->extractQueueItemId($message); if (!$queueItemId) { return false; diff --git a/application/Espo/Core/Mail/Message.php b/application/Espo/Core/Mail/Message.php index 38c7587471..d753e1b1d4 100644 --- a/application/Espo/Core/Mail/Message.php +++ b/application/Espo/Core/Mail/Message.php @@ -29,6 +29,8 @@ namespace Espo\Core\Mail; +use Espo\Core\Mail\Message\Part; + interface Message { /** @@ -67,4 +69,9 @@ interface Message * Whether contents is fetched. */ public function isFetched(): bool; + + /** + * @return Part[] + */ + public function getPartList(): array; } diff --git a/application/Espo/Core/Mail/Message/MailMimeParser/Part.php b/application/Espo/Core/Mail/Message/MailMimeParser/Part.php new file mode 100644 index 0000000000..3abff1a730 --- /dev/null +++ b/application/Espo/Core/Mail/Message/MailMimeParser/Part.php @@ -0,0 +1,79 @@ +part = $part; + } + + public function getContentType(): ?string + { + return $this->part->getContentType(); + } + + public function hasContent(): bool + { + return $this->part->hasContent(); + } + + public function getContent(): ?string + { + return $this->part->getContent(); + } + + public function getContentId(): ?string + { + return $this->part->getContentId(); + } + + public function getCharset(): ?string + { + return $this->part->getCharset(); + } + + public function getContentDisposition(): ?string + { + return $this->part->getContentDisposition(); + } + + public function getFilename(): ?string + { + return $this->part->getFilename(); + } +} diff --git a/application/Espo/Core/Mail/Message/Part.php b/application/Espo/Core/Mail/Message/Part.php new file mode 100644 index 0000000000..3c5ab2fffb --- /dev/null +++ b/application/Espo/Core/Mail/Message/Part.php @@ -0,0 +1,47 @@ +storage = $storage; $this->parser = $parser; $this->fullRawContent = $fullRawContent; + + if ( + !$storage && + $this->fullRawContent && + strpos($this->fullRawContent, "\r\n\r\n") !== false + ) { + [$rawHeader, $rawBody] = explode("\r\n\r\n", $this->fullRawContent, 2); + + $this->rawHeader = $rawHeader; + $this->rawContent = $rawBody; + } } public function getRawHeader(): string @@ -133,4 +145,16 @@ class MessageWrapper implements Message { return (bool) $this->rawHeader; } + + /** + * @return Part[] + */ + public function getPartList(): array + { + if (!$this->parser) { + throw new RuntimeException(); + } + + return $this->parser->getPartList($this); + } } diff --git a/application/Espo/Core/Mail/Parser.php b/application/Espo/Core/Mail/Parser.php index f719694fda..9e65cffa5b 100644 --- a/application/Espo/Core/Mail/Parser.php +++ b/application/Espo/Core/Mail/Parser.php @@ -32,6 +32,7 @@ namespace Espo\Core\Mail; use Espo\Entities\Email; use Espo\Entities\Attachment; use Espo\Core\Mail\Message; +use Espo\Core\Mail\Message\Part; use stdClass; @@ -56,4 +57,9 @@ interface Parser * @return Attachment[] A list of inline attachments. */ public function getInlineAttachmentList(Message $message, Email $email): array; + + /** + * @return Part[] + */ + public function getPartList(Message $message): array; } diff --git a/application/Espo/Core/Mail/Parsers/MailMimeParser.php b/application/Espo/Core/Mail/Parsers/MailMimeParser.php index a4ec4aa681..6ee64275b6 100644 --- a/application/Espo/Core/Mail/Parsers/MailMimeParser.php +++ b/application/Espo/Core/Mail/Parsers/MailMimeParser.php @@ -36,6 +36,10 @@ use Espo\Entities\Attachment; use Espo\Core\Mail\Message; use Espo\Core\Mail\Parser; +use Espo\Core\Mail\Message\Part; + +use Espo\Core\Mail\Message\MailMimeParser\Part as WrapperPart; + use Espo\ORM\EntityManager; use ZBateson\MailMimeParser\MailMimeParser as WrappeeParser; @@ -211,6 +215,22 @@ class MailMimeParser implements Parser return $addressList; } + /** + * @return Part[] + */ + public function getPartList(Message $message): array + { + $wrappeeList = $this->getMessage($message)->getChildParts(); + + $partList = []; + + foreach ($wrappeeList as $wrappee) { + $partList[] = new WrapperPart($wrappee); + } + + return $partList; + } + /** * @return Attachment[] */ diff --git a/tests/unit/Espo/Core/Mail/Account/BouncedRecognizerTest.php b/tests/unit/Espo/Core/Mail/Account/BouncedRecognizerTest.php new file mode 100644 index 0000000000..972f0b1594 --- /dev/null +++ b/tests/unit/Espo/Core/Mail/Account/BouncedRecognizerTest.php @@ -0,0 +1,96 @@ +bouncedRecognizer = new BouncedRecognizer(); + } + + private function createMessage(string $contents): MessageWrapper + { + $entityManager = $this->createMock(EntityManager::class); + + $parser = new MailMimeParser($entityManager); + + return new MessageWrapper(0, null, $parser, $contents); + } + + public function testBounced1a(): void + { + $contents = file_get_contents('tests/unit/testData/Core/Mail/bounced_1.eml'); + + $message = $this->createMessage($contents); + + $this->assertTrue($this->bouncedRecognizer->isBounced($message)); + $this->assertTrue($this->bouncedRecognizer->isHard($message)); + $this->assertEquals('0011aa', $this->bouncedRecognizer->extractQueueItemId($message)); + } + + public function testBounced1b(): void + { + $contents = file_get_contents('tests/unit/testData/Core/Mail/bounced_1.eml'); + $contents = str_replace('MAILER-DAEMON', 'test', $contents); + + $message = $this->createMessage($contents); + + $this->assertTrue($this->bouncedRecognizer->isBounced($message)); + $this->assertTrue($this->bouncedRecognizer->isHard($message)); + } + + public function testBounced2(): void + { + $contents = file_get_contents('tests/unit/testData/Core/Mail/bounced_2.eml'); + + $message = $this->createMessage($contents); + + $this->assertTrue($this->bouncedRecognizer->isBounced($message)); + $this->assertFalse($this->bouncedRecognizer->isHard($message)); + } + + public function testNotBounced1(): void + { + $contents = file_get_contents('tests/unit/testData/Core/Mail/test_email_1.eml'); + + $message = $this->createMessage($contents); + + $this->assertFalse($this->bouncedRecognizer->isBounced($message)); + } +} diff --git a/tests/unit/testData/Core/Mail/bounced_1.eml b/tests/unit/testData/Core/Mail/bounced_1.eml new file mode 100644 index 0000000000..5b836b97aa --- /dev/null +++ b/tests/unit/testData/Core/Mail/bounced_1.eml @@ -0,0 +1,65 @@ +Delivered-To: me@test.com +Received: by 10.213.9.17 with SMTP id j17csp4766ebj; + Tue, 8 May 2012 20:34:50 -0700 (PDT) +Received: by 10.182.207.10 with SMTP id ls10mr30733797obc.9.1336534489928; + Tue, 08 May 2012 20:34:49 -0700 (PDT) +Return-Path: <> +Received: from mx3.name.com (mx3.name.com. [173.192.7.98]) + by mx.google.com with ESMTP id s3si768112obn.1.2012.05.08.20.34.49; + Tue, 08 May 2012 20:34:49 -0700 (PDT) +Received-SPF: pass (google.com: best guess record for domain of mx3.name.com designates 173.192.7.98 as permitted sender) client-ip=173.192.7.98; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of mx3.name.com designates 173.192.7.98 as permitted sender) smtp.mail= +Received-SPF: None (no SPF record) identity=helo; client-ip=199.255.192.13; helo=a192-13.smtp-out.amazonses.com; envelope-from=<>; receiver=info@test.com +Received: from a192-13.smtp-out.amazonses.com (a192-13.smtp-out.amazonses.com [199.255.192.13]) + by mx3.name.com (Postfix) with ESMTP id 5972F6000074C + for ; Tue, 8 May 2012 22:34:49 -0500 (CDT) +X-Original-To: 000001372d0dbf88-76d26d51-9d96-468a-9071-318ba2c35003-000000@amazonses.com +Delivered-To: 000001372d0dbf88-76d26d51-9d96-468a-9071-318ba2c35003-000000@amazonses.com +Message-Id: <000001372fa9d596-ec772006-9987-11e1-8d9b-433290f94ba3-000000@email.amazonses.com> +Date: Wed, 9 May 2012 03:34:48 +0000 +To: info+bounce-qid-0011aa@test.com +From: MAILER-DAEMON@amazonses.com +Subject: Delivery Status Notification (Failure) +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; boundary="ACnzx.4lvBb052b.srYYd.CyGnkMZ" +X-AWS-Outgoing: 199.255.192.13 + +--ACnzx.4lvBb052b.srYYd.CyGnkMZ +content-type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +The following message to was undeliverable. +The reason for the problem: +5.4.7 - Delivery expired (message too old) 'no valid ip addresses' + +--ACnzx.4lvBb052b.srYYd.CyGnkMZ +content-type: message/delivery-status + +Reporting-MTA: dns; na-mm-outgoing-7101-bacon.iad7.amazon.com + +Final-Recipient: rfc822;wrongster@test.com +Action: failed +Status: 5.0.0 (permanent failure) +Diagnostic-Code: smtp; 5.4.7 - Delivery expired (message too old) 'no valid ip addresses' (delivery attempts: 0) + +--ACnzx.4lvBb052b.srYYd.CyGnkMZ +content-type: message/rfc822 + +Received: from unknown (HELO aws-bacon-dlvr-svc-na-i-986ddefa.us-east-1.amazon.com) ([10.13.133.79]) + by na-mm-outgoing-7101-bacon.iad7.amazon.com with ESMTP; 08 May 2012 16:15:55 +0000 +Return-Path: 000001372d0dbf88-76d26d51-9d96-468a-9071-318ba2c35003-000000@amazonses.com +Date: Tue, 8 May 2012 15:25:04 +0000 +From: info@test.com +To: wrongster@test.com +Message-ID: <000001372d0dbf88-76d26d51-9d96-468a-9071-318ba2c35003-000000@email.amazonses.com> +Subject: Test subject +Mime-Version: 1.0 +Content-Type: text/plain; + charset=UTF-8 +Content-Transfer-Encoding: 7bit +X-AWS-Outgoing: 199.127.232.7 + +Test message. + +--ACnzx.4lvBb052b.srYYd.CyGnkMZ-- \ No newline at end of file diff --git a/tests/unit/testData/Core/Mail/bounced_2.eml b/tests/unit/testData/Core/Mail/bounced_2.eml new file mode 100644 index 0000000000..579fa7d41b --- /dev/null +++ b/tests/unit/testData/Core/Mail/bounced_2.eml @@ -0,0 +1,41 @@ +Date: Thu, 7 Jul 1994 17:16:05 -0400 +From: Mail Delivery Subsystem + Message-Id: +<199407072116.RAA14128@CS.UTK.EDU> +Subject: Returned mail: Cannot +send message for 5 days +To: MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; boundary="RAA14128.773615765/CS.UTK.EDU" + +--RAA14128.773615765/CS.UTK.EDU + +The original message was received at Sat, 2 Jul 1994 17:10:28 -0400 +from root@localhost + + ----- The following addresses had delivery problems ----- + (unrecoverable error) + +----- Transcript of session follows ----- +... Deferred: Connection timed out + with larry.slip.umd.edu. +Message could not be delivered for 5 days +Message will be deleted from queue + +--RAA14128.773615765/CS.UTK.EDU +content-type: message/delivery-status + +Reporting-MTA: dns; cs.utk.edu + +Original-Recipient: rfc822;louisl@larry.slip.umd.edu +Final-Recipient: rfc822;louisl@larry.slip.umd.edu +Action: failed +Status: 4.0.0 +Diagnostic-Code: smtp; 426 connection timed out +Last-Attempt-Date: Thu, 7 Jul 1994 17:15:49 -0400 + +--RAA14128.773615765/CS.UTK.EDU +content-type: message/rfc822 + +[original message goes here] + +--RAA14128.773615765/CS.UTK.EDU-- \ No newline at end of file