From a33e10e92d1eae17ebcbe63f10f26bf1b62f76e7 Mon Sep 17 00:00:00 2001 From: yuri Date: Mon, 13 Mar 2017 16:22:25 +0200 Subject: [PATCH] mailmimeparser and email parsing imrovements --- application/Espo/Core/Mail/Importer.php | 15 +- .../Espo/Core/Mail/Parsers/MailMimeParser.php | 222 ++++++++++++++++++ .../Core/Mail/Parsers/PhpMimeMailParser.php | 41 +++- .../Espo/Core/Mail/Parsers/ZendMail.php | 46 ++++ .../Resources/metadata/entityDefs/Email.json | 5 + application/Espo/Services/EmailAccount.php | 23 +- application/Espo/Services/InboundEmail.php | 23 +- composer.json | 3 +- composer.lock | 55 ++++- 9 files changed, 388 insertions(+), 45 deletions(-) create mode 100644 application/Espo/Core/Mail/Parsers/MailMimeParser.php diff --git a/application/Espo/Core/Mail/Importer.php b/application/Espo/Core/Mail/Importer.php index 9b426996f5..b294137568 100644 --- a/application/Espo/Core/Mail/Importer.php +++ b/application/Espo/Core/Mail/Importer.php @@ -104,12 +104,16 @@ class Importer } } - if ($parser->checkMessageAttribute($message, 'from')) { - $email->set('fromString', $parser->getMessageAttribute($message, 'from')); + $fromAddressData = $parser->getAddressDataFromMessage($message, 'from'); + if ($fromAddressData) { + $fromString = ($fromAddressData['name'] ? ($fromAddressData['name'] . ' ') : '') . '<' . $fromAddressData['address'] .'>'; + $email->set('fromString', $fromString); } - if ($parser->checkMessageAttribute($message, 'reply-To')) { - $email->set('replyToString', $parser->getMessageAttribute($message, 'reply-To')); + $replyToData = $parser->getAddressDataFromMessage($message, 'reply-To'); + if ($replyToData) { + $replyToString = ($replyToData['name'] ? ($replyToData['name'] . ' ') : '') . '<' . $replyToData['address'] .'>'; + $email->set('replyToString', $replyToString); } $fromArr = $parser->getAddressListFromMessage($message, 'from'); @@ -124,6 +128,9 @@ class Importer $email->set('cc', implode(';', $ccArr)); $email->set('replyTo', implode(';', $replyToArr)); + $addressNameMap = $parser->getAddressNameMap($message); + $email->set('addressNameMap', $addressNameMap); + if ($folderData) { foreach ($folderData as $uId => $folderId) { $email->setLinkMultipleColumn('users', 'folderId', $uId, $folderId); diff --git a/application/Espo/Core/Mail/Parsers/MailMimeParser.php b/application/Espo/Core/Mail/Parsers/MailMimeParser.php new file mode 100644 index 0000000000..d8ebddb76a --- /dev/null +++ b/application/Espo/Core/Mail/Parsers/MailMimeParser.php @@ -0,0 +1,222 @@ +entityManager = $entityManager; + } + + protected function getEntityManager() + { + return $this->entityManager; + } + + protected function getParser() + { + if (!$this->parser) { + $this->parser = new \ZBateson\MailMimeParser\MailMimeParser(); + } + + return $this->parser; + } + + protected function loadContent($message) + { + $raw = $message->getFullRawContent(); + $key = spl_object_hash($message); + $this->messageHash[$key] = $this->getParser()->parse($raw); + } + + protected function getMessage($message) + { + $key = spl_object_hash($message); + if (!array_key_exists($key, $this->messageHash)) { + $raw = $message->getRawHeader(); + if (!$raw) { + $raw = $message->getFullRawContent(); + } + $this->messageHash[$key] = $this->getParser()->parse($raw); + } + + return $this->messageHash[$key]; + } + + public function checkMessageAttribute($message, $attribute) + { + return $this->getMessage($message)->getHeaderValue($attribute) !== null; + } + + public function getMessageAttribute($message, $attribute) + { + if (!$this->checkMessageAttribute($message, $attribute)) return null; + + return $this->getMessage($message)->getHeaderValue($attribute); + } + + public function getMessageMessageId($message) + { + return $this->getMessageAttribute($message, 'Message-ID'); + } + + public function getAddressNameMap($message) + { + $map = (object) []; + + foreach (['from', 'to', 'cc', 'reply-To'] as $type) { + $header = $this->getMessage($message)->getHeader($type); + if ($header) { + $list = $header->getAddresses(); + foreach ($list as $item) { + $address = $item->getEmail(); + $name = $item->getName(); + if ($name && $address) { + $map->$address = $name; + } + } + } + } + + return $map; + } + + public function getAddressDataFromMessage($message, $type) + { + $addressList = []; + $header = $this->getMessage($message)->getHeader($type); + if ($header) { + foreach ($header->getAddresses() as $item) { + return [ + 'address' => $item->getEmail(), + 'name' => $item->getName() + ]; + } + } + return null; + } + + public function getAddressListFromMessage($message, $type) + { + $addressList = []; + $header = $this->getMessage($message)->getHeader($type); + if ($header) { + $list = $header->getAddresses(); + foreach ($list as $address) { + $addressList[] = $address->getEmail(); + } + } + return $addressList; + } + + public function fetchContentParts(\Espo\Entities\Email $email, $message) + { + $this->loadContent($message); + + $bodyPlain = $this->getMessage($message)->getTextContent(); + $bodyHtml = $this->getMessage($message)->getHtmlContent(); + + if ($bodyHtml) { + $email->set('isHtml', true); + $email->set('body', $bodyHtml); + $email->set('bodyPlain', $bodyPlain); + } else { + $email->set('isHtml', false); + $email->set('body', $bodyPlain); + } + + if (!$email->get('body') && $email->get('bodyPlain')) { + $email->set('body', $email->get('bodyPlain')); + } + + $attachmentObjList = $this->getMessage($message)->getAllAttachmentParts(); + $inlineIds = array(); + + foreach ($attachmentObjList as $attachmentObj) { + $attachment = $this->getEntityManager()->getEntity('Attachment'); + + $content = $attachmentObj->getContent(); + + $disposition = $attachmentObj->getHeaderValue('Content-Disposition'); + + $attachment = $this->getEntityManager()->getEntity('Attachment'); + $attachment->set('name', $attachmentObj->getHeaderParameter('Content-Disposition', 'filename', 'unnamed')); + $attachment->set('type', $attachmentObj->getHeaderValue('Content-Type')); + + $contentId = $attachmentObj->getHeaderValue('Content-ID'); + + if ($contentId) { + $contentId = trim($contentId, '<>'); + } + + if ($disposition == 'inline') { + $attachment->set('role', 'Inline Attachment'); + } else { + $attachment->set('role', 'Attachment'); + } + + $attachment->set('contents', $content); + + $this->getEntityManager()->saveEntity($attachment); + + if ($disposition == 'attachment') { + $email->addLinkMultipleId('attachments', $attachment->id); + if ($contentId) { + $inlineIds[$contentId] = $attachment->id; + } + } else if ($disposition == 'inline') { + if ($contentId) { + $inlineIds[$contentId] = $attachment->id; + } + } + } + + $body = $email->get('body'); + + if (!empty($body)) { + foreach ($inlineIds as $cid => $attachmentId) { + if (strpos($body, 'cid:' . $cid) !== false) { + $body = str_replace('cid:' . $cid, '?entryPoint=attachment&id=' . $attachmentId, $body); + } else { + $email->addLinkMultipleId('attachments', $attachmentId); + } + } + $email->set('body', $body); + } + } +} + diff --git a/application/Espo/Core/Mail/Parsers/PhpMimeMailParser.php b/application/Espo/Core/Mail/Parsers/PhpMimeMailParser.php index 99654ed648..db8e0de91c 100644 --- a/application/Espo/Core/Mail/Parsers/PhpMimeMailParser.php +++ b/application/Espo/Core/Mail/Parsers/PhpMimeMailParser.php @@ -85,6 +85,42 @@ class PhpMimeMailParser return $this->getMessageAttribute($message, 'Message-ID'); } + public function getAddressNameMap($message) + { + $map = (object) []; + + foreach (['from', 'to', 'cc', 'reply-To'] as $type) { + if ($this->checkMessageAttribute($message, $type)) { + $list = $this->getParser($message)->getAddresses($type); + foreach ($list as $item) { + $name = $list[0]['display']; + $address = $list[0]['address']; + if ($name && $address && $name !== $address) { + $map->$address = $name; + } + } + + } + } + + return $map; + } + + public function getAddressDataFromMessage($message, $type) + { + $addressList = []; + if ($this->checkMessageAttribute($message, $type)) { + $list = $this->getParser($message)->getAddresses($type); + if (count($list)) { + return [ + 'address' => $list[0]['address'], + 'name' => $list[0]['display'], + ]; + } + } + return null; + } + public function getAddressListFromMessage($message, $type) { $addressList = []; @@ -144,12 +180,13 @@ class PhpMimeMailParser if ($disposition == 'attachment') { $email->addLinkMultipleId('attachments', $attachment->id); - if ($contentId) { $inlineIds[$contentId] = $attachment->id; } } else if ($disposition == 'inline') { - $inlineIds[$contentId] = $attachment->id; + if ($contentId) { + $inlineIds[$contentId] = $attachment->id; + } } } diff --git a/application/Espo/Core/Mail/Parsers/ZendMail.php b/application/Espo/Core/Mail/Parsers/ZendMail.php index e312e028dc..9ec30c7135 100644 --- a/application/Espo/Core/Mail/Parsers/ZendMail.php +++ b/application/Espo/Core/Mail/Parsers/ZendMail.php @@ -72,6 +72,45 @@ class ZendMail return $messageId; } + public function getAddressNameMap($message) + { + $map = (object) []; + + $zendMessage = $message->getZendMessage(); + + foreach (['from', 'to', 'cc', 'reply-To'] as $type) { + if (isset($zendMessage->$type)) { + $list = $this->normilizeHeader($zendMessage->getHeader($type))->getAddressList(); + foreach ($list as $item) { + $name = $item->getName(); + $address = $item->getEmail(); + if ($name && $address && $name !== $address) { + $map->$address = $name; + } + } + } + } + + return $map; + } + + public function getAddressDataFromMessage($message, $type) + { + $zendMessage = $message->getZendMessage(); + + $addressList = array(); + if (isset($zendMessage->$type)) { + $list = $this->normilizeHeader($zendMessage->getHeader($type))->getAddressList(); + foreach ($list as $address) { + return [ + 'address' => $address->getEmail(), + 'name' => $address->getName() + ]; + } + } + return null; + } + public function getAddressListFromMessage($message, $type) { $zendMessage = $message->getZendMessage(); @@ -227,6 +266,13 @@ class ZendMail $attachmentsIds = $email->get('attachmentsIds'); $attachmentsIds[] = $attachment->id; $email->set('attachmentsIds', $attachmentsIds); + + if (isset($part->contentID)) { + $contentId = trim($part->contentID, '<>'); + if ($contentId) { + $inlineIds[$contentId] = $attachment->id; + } + } } else if ($disposition == 'inline') { $inlineIds[$contentId] = $attachment->id; } diff --git a/application/Espo/Resources/metadata/entityDefs/Email.json b/application/Espo/Resources/metadata/entityDefs/Email.json index 94c5c00289..b5dfb866e0 100644 --- a/application/Espo/Resources/metadata/entityDefs/Email.json +++ b/application/Espo/Resources/metadata/entityDefs/Email.json @@ -22,6 +22,11 @@ "replyToString": { "type": "varchar" }, + "addressNameMap": { + "type": "jsonObject", + "disabled": true, + "readOnly": true + }, "from": { "type": "varchar", "notStorable": true, diff --git a/application/Espo/Services/EmailAccount.php b/application/Espo/Services/EmailAccount.php index 46961bc629..b776f1c6d8 100644 --- a/application/Espo/Services/EmailAccount.php +++ b/application/Espo/Services/EmailAccount.php @@ -225,7 +225,7 @@ class EmailAccount extends Record $portionLimit = $this->getConfig()->get('personalEmailMaxPortionSize', self::PORTION_LIMIT); - $parserName = 'ZendMail'; + $parserName = 'MailMimeParser'; if (extension_loaded('mailparse')) { $parserName = 'PhpMimeMailParser'; } @@ -306,9 +306,7 @@ class EmailAccount extends Record $flags = $message->getFlags(); } - $importMethodName = 'importWith' . $parserName; - - $email = $this->$importMethodName($importer, $emailAccount, $message, $teamIdList, null, [$userId], $filterCollection, $fetchOnlyHeader, $folderData); + $email = $this->importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, null, [$userId], $filterCollection, $fetchOnlyHeader, $folderData); if ($emailAccount->get('keepFetchedEmailsUnread')) { if (is_array($flags) && empty($flags[Storage::FLAG_SEEN])) { @@ -362,24 +360,13 @@ class EmailAccount extends Record return true; } - protected function importWithPhpMimeMailParser($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null) + protected function importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null) { $email = null; try { - $email = $importer->importMessage('PhpMimeMailParser', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData); + $email = $importer->importMessage($parserName, $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData); } catch (\Exception $e) { - $GLOBALS['log']->error('EmailAccount '.$emailAccount->id.' (Import Message w/ php-mime-mail-parser): [' . $e->getCode() . '] ' .$e->getMessage()); - } - return $email; - } - - protected function importWithZendMail($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null) - { - $email = null; - try { - $email = $importer->importMessage('ZendMail', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData); - } catch (\Exception $e) { - $GLOBALS['log']->error('EmailAccount '.$emailAccount->id.' (Import Message w/ zend-mail): [' . $e->getCode() . '] ' .$e->getMessage()); + $GLOBALS['log']->error('EmailAccount '.$emailAccount->id.' (Import Message w/ '.$parserName.'): [' . $e->getCode() . '] ' .$e->getMessage()); } return $email; } diff --git a/application/Espo/Services/InboundEmail.php b/application/Espo/Services/InboundEmail.php index a7109e14f2..f66c3c0751 100644 --- a/application/Espo/Services/InboundEmail.php +++ b/application/Espo/Services/InboundEmail.php @@ -232,7 +232,7 @@ class InboundEmail extends \Espo\Services\Record $monitoredFolders = 'INBOX'; } - $parserName = 'ZendMail'; + $parserName = 'MailMimeParser'; if (extension_loaded('mailparse')) { $parserName = 'PhpMimeMailParser'; } @@ -320,9 +320,7 @@ class InboundEmail extends \Espo\Services\Record } } if (!$toSkip) { - $importMethodName = 'importWith' . $parserName; - - $email = $this->$importMethodName($importer, $emailAccount, $message, $teamIdList, $userId, $userIdList, $filterCollection, $fetchOnlyHeader, null); + $email = $this->importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, $userId, $userIdList, $filterCollection, $fetchOnlyHeader, null); } } catch (\Exception $e) { $GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Get Message w/ parser '.$parserName.'): [' . $e->getCode() . '] ' .$e->getMessage()); @@ -386,24 +384,13 @@ class InboundEmail extends \Espo\Services\Record return true; } - protected function importWithPhpMimeMailParser($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null) + protected function importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null) { $email = null; try { - $email = $importer->importMessage('PhpMimeMailParser', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData); + $email = $importer->importMessage($parserName, $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData); } catch (\Exception $e) { - $GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Import Message w/ php-mime-mail-parser): [' . $e->getCode() . '] ' .$e->getMessage()); - } - return $email; - } - - protected function importWithZendMail($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null) - { - $email = null; - try { - $email = $importer->importMessage('ZendMail', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData); - } catch (\Exception $e) { - $GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Import Message w/ zend-mail): [' . $e->getCode() . '] ' .$e->getMessage()); + $GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Import Message w/ '.$parserName.'): [' . $e->getCode() . '] ' .$e->getMessage()); } return $email; } diff --git a/composer.json b/composer.json index 15ddd0f656..904bd42863 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,8 @@ "composer/semver": "^1.4", "zendframework/zend-servicemanager": "2.6.0", "tecnickcom/tcpdf": "^6.2", - "php-mime-mail-parser/php-mime-mail-parser": "^2.5" + "php-mime-mail-parser/php-mime-mail-parser": "^2.5", + "zbateson/mail-mime-parser": "^0.4.1" }, "autoload": { "psr-0": { diff --git a/composer.lock b/composer.lock index 03a7e476c0..e20d166c0e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,8 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "e599a4ab425b29d1e585685684a0bffd", - "content-hash": "81595e763c766d29eca4a5b03a007f2f", + "hash": "6e1e3327b6081f402280ed7ad0c324f3", + "content-hash": "c98074c42af6b9e3396ae058bc602bea", "packages": [ { "name": "composer/semver", @@ -963,6 +963,57 @@ ], "time": "2014-07-13 09:19:12" }, + { + "name": "zbateson/mail-mime-parser", + "version": "0.4.1", + "source": { + "type": "git", + "url": "https://github.com/zbateson/MailMimeParser.git", + "reference": "63dec759b5933cd858302e058ebb954f7c48e522" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/zbateson/MailMimeParser/zipball/63dec759b5933cd858302e058ebb954f7c48e522", + "reference": "63dec759b5933cd858302e058ebb954f7c48e522", + "shasum": "" + }, + "require": { + "ext-mbstring": "*", + "php": ">=5.4" + }, + "require-dev": { + "evert/phpdoc-md": "~0.1.1", + "phing/phing": "2.*", + "phpdocumentor/phpdocumentor": "2.*", + "phpunit/phpunit": "~4.8" + }, + "type": "library", + "autoload": { + "psr-4": { + "ZBateson\\MailMimeParser\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-2-Clause" + ], + "authors": [ + { + "name": "Zaahid Bateson", + "email": "zbateson@users.noreply.github.com" + } + ], + "description": "MIME email message parser", + "homepage": "https://github.com/zbateson/MailMimeParser", + "keywords": [ + "email", + "mail", + "mime", + "parser", + "php-imap" + ], + "time": "2017-03-05 22:50:58" + }, { "name": "zendframework/zend-crypt", "version": "2.6.0",