espocrm-extensions/espocrm-base
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mailmimeparser and email parsing imrovements

Browse Source
This commit is contained in:
yuri
2017-03-13 16:22:25 +02:00
parent 86631117a2
commit a33e10e92d
9 changed files with 388 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
application/Espo/Core/Mail/Importer.php
+11 -4
View File
@@ -104,12 +104,16 @@ class Importer
}
}
if ($parser->checkMessageAttribute($message, 'from')) {
$email->set('fromString', $parser->getMessageAttribute($message, 'from'));
$fromAddressData = $parser->getAddressDataFromMessage($message, 'from');
if ($fromAddressData) {
$fromString = ($fromAddressData['name'] ? ($fromAddressData['name'] . ' ') : '') . '<' . $fromAddressData['address'] .'>';
$email->set('fromString', $fromString);
}
if ($parser->checkMessageAttribute($message, 'reply-To')) {
$email->set('replyToString', $parser->getMessageAttribute($message, 'reply-To'));
$replyToData = $parser->getAddressDataFromMessage($message, 'reply-To');
if ($replyToData) {
$replyToString = ($replyToData['name'] ? ($replyToData['name'] . ' ') : '') . '<' . $replyToData['address'] .'>';
$email->set('replyToString', $replyToString);
}
$fromArr = $parser->getAddressListFromMessage($message, 'from');
@@ -124,6 +128,9 @@ class Importer
$email->set('cc', implode(';', $ccArr));
$email->set('replyTo', implode(';', $replyToArr));
$addressNameMap = $parser->getAddressNameMap($message);
$email->set('addressNameMap', $addressNameMap);
if ($folderData) {
foreach ($folderData as $uId => $folderId) {
$email->setLinkMultipleColumn('users', 'folderId', $uId, $folderId);
application/Espo/Core/Mail/Parsers/MailMimeParser.php
+222
View File
@@ -0,0 +1,222 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM - Open Source CRM application.
* Copyright (C) 2014-2017 Yuri Kuznetsov, Taras Machyshyn, Oleksiy Avramenko
* Website: http://www.espocrm.com
*
* EspoCRM is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* EspoCRM is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with EspoCRM. If not, see http://www.gnu.org/licenses/.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU General Public License version 3.
*
* In accordance with Section 7(b) of the GNU General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Mail\Parsers;
class MailMimeParser
{
private $entityManager;
private $parser = array();
protected $messageHash = array();
public function __construct($entityManager)
{
$this->entityManager = $entityManager;
}
protected function getEntityManager()
{
return $this->entityManager;
}
protected function getParser()
{
if (!$this->parser) {
$this->parser = new \ZBateson\MailMimeParser\MailMimeParser();
}
return $this->parser;
}
protected function loadContent($message)
{
$raw = $message->getFullRawContent();
$key = spl_object_hash($message);
$this->messageHash[$key] = $this->getParser()->parse($raw);
}
protected function getMessage($message)
{
$key = spl_object_hash($message);
if (!array_key_exists($key, $this->messageHash)) {
$raw = $message->getRawHeader();
if (!$raw) {
$raw = $message->getFullRawContent();
}
$this->messageHash[$key] = $this->getParser()->parse($raw);
}
return $this->messageHash[$key];
}
public function checkMessageAttribute($message, $attribute)
{
return $this->getMessage($message)->getHeaderValue($attribute) !== null;
}
public function getMessageAttribute($message, $attribute)
{
if (!$this->checkMessageAttribute($message, $attribute)) return null;
return $this->getMessage($message)->getHeaderValue($attribute);
}
public function getMessageMessageId($message)
{
return $this->getMessageAttribute($message, 'Message-ID');
}
public function getAddressNameMap($message)
{
$map = (object) [];
foreach (['from', 'to', 'cc', 'reply-To'] as $type) {
$header = $this->getMessage($message)->getHeader($type);
if ($header) {
$list = $header->getAddresses();
foreach ($list as $item) {
$address = $item->getEmail();
$name = $item->getName();
if ($name && $address) {
$map->$address = $name;
}
}
}
}
return $map;
}
public function getAddressDataFromMessage($message, $type)
{
$addressList = [];
$header = $this->getMessage($message)->getHeader($type);
if ($header) {
foreach ($header->getAddresses() as $item) {
return [
'address' => $item->getEmail(),
'name' => $item->getName()
];
}
}
return null;
}
public function getAddressListFromMessage($message, $type)
{
$addressList = [];
$header = $this->getMessage($message)->getHeader($type);
if ($header) {
$list = $header->getAddresses();
foreach ($list as $address) {
$addressList[] = $address->getEmail();
}
}
return $addressList;
}
public function fetchContentParts(\Espo\Entities\Email $email, $message)
{
$this->loadContent($message);
$bodyPlain = $this->getMessage($message)->getTextContent();
$bodyHtml = $this->getMessage($message)->getHtmlContent();
if ($bodyHtml) {
$email->set('isHtml', true);
$email->set('body', $bodyHtml);
$email->set('bodyPlain', $bodyPlain);
} else {
$email->set('isHtml', false);
$email->set('body', $bodyPlain);
}
if (!$email->get('body') && $email->get('bodyPlain')) {
$email->set('body', $email->get('bodyPlain'));
}
$attachmentObjList = $this->getMessage($message)->getAllAttachmentParts();
$inlineIds = array();
foreach ($attachmentObjList as $attachmentObj) {
$attachment = $this->getEntityManager()->getEntity('Attachment');
$content = $attachmentObj->getContent();
$disposition = $attachmentObj->getHeaderValue('Content-Disposition');
$attachment = $this->getEntityManager()->getEntity('Attachment');
$attachment->set('name', $attachmentObj->getHeaderParameter('Content-Disposition', 'filename', 'unnamed'));
$attachment->set('type', $attachmentObj->getHeaderValue('Content-Type'));
$contentId = $attachmentObj->getHeaderValue('Content-ID');
if ($contentId) {
$contentId = trim($contentId, '<>');
}
if ($disposition == 'inline') {
$attachment->set('role', 'Inline Attachment');
} else {
$attachment->set('role', 'Attachment');
}
$attachment->set('contents', $content);
$this->getEntityManager()->saveEntity($attachment);
if ($disposition == 'attachment') {
$email->addLinkMultipleId('attachments', $attachment->id);
if ($contentId) {
$inlineIds[$contentId] = $attachment->id;
}
} else if ($disposition == 'inline') {
if ($contentId) {
$inlineIds[$contentId] = $attachment->id;
}
}
}
$body = $email->get('body');
if (!empty($body)) {
foreach ($inlineIds as $cid => $attachmentId) {
if (strpos($body, 'cid:' . $cid) !== false) {
$body = str_replace('cid:' . $cid, '?entryPoint=attachment&amp;id=' . $attachmentId, $body);
} else {
$email->addLinkMultipleId('attachments', $attachmentId);
}
}
$email->set('body', $body);
}
}
}
application/Espo/Core/Mail/Parsers/PhpMimeMailParser.php
+39 -2
View File
@@ -85,6 +85,42 @@ class PhpMimeMailParser
return $this->getMessageAttribute($message, 'Message-ID');
}
public function getAddressNameMap($message)
{
$map = (object) [];
foreach (['from', 'to', 'cc', 'reply-To'] as $type) {
if ($this->checkMessageAttribute($message, $type)) {
$list = $this->getParser($message)->getAddresses($type);
foreach ($list as $item) {
$name = $list[0]['display'];
$address = $list[0]['address'];
if ($name && $address && $name !== $address) {
$map->$address = $name;
}
}
}
}
return $map;
}
public function getAddressDataFromMessage($message, $type)
{
$addressList = [];
if ($this->checkMessageAttribute($message, $type)) {
$list = $this->getParser($message)->getAddresses($type);
if (count($list)) {
return [
'address' => $list[0]['address'],
'name' => $list[0]['display'],
];
}
}
return null;
}
public function getAddressListFromMessage($message, $type)
{
$addressList = [];
@@ -144,12 +180,13 @@ class PhpMimeMailParser
if ($disposition == 'attachment') {
$email->addLinkMultipleId('attachments', $attachment->id);
if ($contentId) {
$inlineIds[$contentId] = $attachment->id;
}
} else if ($disposition == 'inline') {
$inlineIds[$contentId] = $attachment->id;
if ($contentId) {
$inlineIds[$contentId] = $attachment->id;
}
}
}
application/Espo/Core/Mail/Parsers/ZendMail.php
+46
View File
@@ -72,6 +72,45 @@ class ZendMail
return $messageId;
}
public function getAddressNameMap($message)
{
$map = (object) [];
$zendMessage = $message->getZendMessage();
foreach (['from', 'to', 'cc', 'reply-To'] as $type) {
if (isset($zendMessage->$type)) {
$list = $this->normilizeHeader($zendMessage->getHeader($type))->getAddressList();
foreach ($list as $item) {
$name = $item->getName();
$address = $item->getEmail();
if ($name && $address && $name !== $address) {
$map->$address = $name;
}
}
}
}
return $map;
}
public function getAddressDataFromMessage($message, $type)
{
$zendMessage = $message->getZendMessage();
$addressList = array();
if (isset($zendMessage->$type)) {
$list = $this->normilizeHeader($zendMessage->getHeader($type))->getAddressList();
foreach ($list as $address) {
return [
'address' => $address->getEmail(),
'name' => $address->getName()
];
}
}
return null;
}
public function getAddressListFromMessage($message, $type)
{
$zendMessage = $message->getZendMessage();
@@ -227,6 +266,13 @@ class ZendMail
$attachmentsIds = $email->get('attachmentsIds');
$attachmentsIds[] = $attachment->id;
$email->set('attachmentsIds', $attachmentsIds);
if (isset($part->contentID)) {
$contentId = trim($part->contentID, '<>');
if ($contentId) {
$inlineIds[$contentId] = $attachment->id;
}
}
} else if ($disposition == 'inline') {
$inlineIds[$contentId] = $attachment->id;
}
application/Espo/Resources/metadata/entityDefs/Email.json
+5
View File
@@ -22,6 +22,11 @@
"replyToString": {
"type": "varchar"
},
"addressNameMap": {
"type": "jsonObject",
"disabled": true,
"readOnly": true
},
"from": {
"type": "varchar",
"notStorable": true,
application/Espo/Services/EmailAccount.php
+5 -18
View File
@@ -225,7 +225,7 @@ class EmailAccount extends Record
$portionLimit = $this->getConfig()->get('personalEmailMaxPortionSize', self::PORTION_LIMIT);
$parserName = 'ZendMail';
$parserName = 'MailMimeParser';
if (extension_loaded('mailparse')) {
$parserName = 'PhpMimeMailParser';
}
@@ -306,9 +306,7 @@ class EmailAccount extends Record
$flags = $message->getFlags();
}
$importMethodName = 'importWith' . $parserName;
$email = $this->$importMethodName($importer, $emailAccount, $message, $teamIdList, null, [$userId], $filterCollection, $fetchOnlyHeader, $folderData);
$email = $this->importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, null, [$userId], $filterCollection, $fetchOnlyHeader, $folderData);
if ($emailAccount->get('keepFetchedEmailsUnread')) {
if (is_array($flags) && empty($flags[Storage::FLAG_SEEN])) {
@@ -362,24 +360,13 @@ class EmailAccount extends Record
return true;
}
protected function importWithPhpMimeMailParser($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null)
protected function importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null)
{
$email = null;
try {
$email = $importer->importMessage('PhpMimeMailParser', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData);
$email = $importer->importMessage($parserName, $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData);
} catch (\Exception $e) {
$GLOBALS['log']->error('EmailAccount '.$emailAccount->id.' (Import Message w/ php-mime-mail-parser): [' . $e->getCode() . '] ' .$e->getMessage());
}
return $email;
}
protected function importWithZendMail($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null)
{
$email = null;
try {
$email = $importer->importMessage('ZendMail', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData);
} catch (\Exception $e) {
$GLOBALS['log']->error('EmailAccount '.$emailAccount->id.' (Import Message w/ zend-mail): [' . $e->getCode() . '] ' .$e->getMessage());
$GLOBALS['log']->error('EmailAccount '.$emailAccount->id.' (Import Message w/ '.$parserName.'): [' . $e->getCode() . '] ' .$e->getMessage());
}
return $email;
}
application/Espo/Services/InboundEmail.php
+5 -18
View File
@@ -232,7 +232,7 @@ class InboundEmail extends \Espo\Services\Record
$monitoredFolders = 'INBOX';
}
$parserName = 'ZendMail';
$parserName = 'MailMimeParser';
if (extension_loaded('mailparse')) {
$parserName = 'PhpMimeMailParser';
}
@@ -320,9 +320,7 @@ class InboundEmail extends \Espo\Services\Record
}
}
if (!$toSkip) {
$importMethodName = 'importWith' . $parserName;
$email = $this->$importMethodName($importer, $emailAccount, $message, $teamIdList, $userId, $userIdList, $filterCollection, $fetchOnlyHeader, null);
$email = $this->importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, $userId, $userIdList, $filterCollection, $fetchOnlyHeader, null);
}
} catch (\Exception $e) {
$GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Get Message w/ parser '.$parserName.'): [' . $e->getCode() . '] ' .$e->getMessage());
@@ -386,24 +384,13 @@ class InboundEmail extends \Espo\Services\Record
return true;
}
protected function importWithPhpMimeMailParser($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null)
protected function importMessage($parserName, $importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null)
{
$email = null;
try {
$email = $importer->importMessage('PhpMimeMailParser', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData);
$email = $importer->importMessage($parserName, $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData);
} catch (\Exception $e) {
$GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Import Message w/ php-mime-mail-parser): [' . $e->getCode() . '] ' .$e->getMessage());
}
return $email;
}
protected function importWithZendMail($importer, $emailAccount, $message, $teamIdList, $userId = null, $userIdList = [], $filterCollection, $fetchOnlyHeader, $folderData = null)
{
$email = null;
try {
$email = $importer->importMessage('ZendMail', $message, $userId, $teamIdList, $userIdList, $filterCollection, $fetchOnlyHeader, $folderData);
} catch (\Exception $e) {
$GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Import Message w/ zend-mail): [' . $e->getCode() . '] ' .$e->getMessage());
$GLOBALS['log']->error('InboundEmail '.$emailAccount->id.' (Import Message w/ '.$parserName.'): [' . $e->getCode() . '] ' .$e->getMessage());
}
return $email;
}
composer.json
+2 -1
View File
@@ -12,7 +12,8 @@
"composer/semver": "^1.4",
"zendframework/zend-servicemanager": "2.6.0",
"tecnickcom/tcpdf": "^6.2",
"php-mime-mail-parser/php-mime-mail-parser": "^2.5"
"php-mime-mail-parser/php-mime-mail-parser": "^2.5",
"zbateson/mail-mime-parser": "^0.4.1"
},
"autoload": {
"psr-0": {
composer.lock
Generated
+53 -2
View File
@@ -4,8 +4,8 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"hash": "e599a4ab425b29d1e585685684a0bffd",
"content-hash": "81595e763c766d29eca4a5b03a007f2f",
"hash": "6e1e3327b6081f402280ed7ad0c324f3",
"content-hash": "c98074c42af6b9e3396ae058bc602bea",
"packages": [
{
"name": "composer/semver",
@@ -963,6 +963,57 @@
],
"time": "2014-07-13 09:19:12"
},
{
"name": "zbateson/mail-mime-parser",
"version": "0.4.1",
"source": {
"type": "git",
"url": "https://github.com/zbateson/MailMimeParser.git",
"reference": "63dec759b5933cd858302e058ebb954f7c48e522"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/zbateson/MailMimeParser/zipball/63dec759b5933cd858302e058ebb954f7c48e522",
"reference": "63dec759b5933cd858302e058ebb954f7c48e522",
"shasum": ""
},
"require": {
"ext-mbstring": "*",
"php": ">=5.4"
},
"require-dev": {
"evert/phpdoc-md": "~0.1.1",
"phing/phing": "2.*",
"phpdocumentor/phpdocumentor": "2.*",
"phpunit/phpunit": "~4.8"
},
"type": "library",
"autoload": {
"psr-4": {
"ZBateson\\MailMimeParser\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"BSD-2-Clause"
],
"authors": [
{
"name": "Zaahid Bateson",
"email": "zbateson@users.noreply.github.com"
}
],
"description": "MIME email message parser",
"homepage": "https://github.com/zbateson/MailMimeParser",
"keywords": [
"email",
"mail",
"mime",
"parser",
"php-imap"
],
"time": "2017-03-05 22:50:58"
},
{
"name": "zendframework/zend-crypt",
"version": "2.6.0",