espocrm-extensions/espocrm-base
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

full text search 2

Browse Source
This commit is contained in:
yuri
2018-06-20 14:51:50 +03:00
parent 775641aee1
commit fa0cb01660
5 changed files with 157 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
application/Espo/Core/DataManager.php
+27 -1
View File
@@ -53,6 +53,8 @@ class DataManager
*/
public function rebuild($entityList = null)
{
$this->populateConfigParameters();
$result = $this->clearCache();
$result &= $this->rebuildMetadata();
@@ -172,5 +174,29 @@ class DataManager
$this->getContainer()->get('config')->save();
return true;
}
}
protected function populateConfigParameters()
{
$config = $this->getContainer()->get('config');
$pdo = $this->getContainer()->get('entityManager')->getPDO();
$query = "SHOW VARIABLES LIKE 'ft_min_word_len'";
$sth = $pdo->prepare($query);
$sth->execute();
$fullTextSearchMinLength = null;
if ($row = $sth->fetch(\PDO::FETCH_ASSOC)) {
if (isset($row['Value'])) {
$fullTextSearchIsNotAvailable = false;
$fullTextSearchMinLength = intval($row['Value']);
}
} else {
$fullTextSearchIsNotAvailable = true;
}
$config->set('fullTextSearchIsNotAvailable', $fullTextSearchIsNotAvailable);
$config->set('fullTextSearchMinLength', $fullTextSearchMinLength);
$config->save();
}
}
application/Espo/Core/SelectManagers/Base.php
+101 -32
View File
@@ -66,6 +66,8 @@ class Base
const MIN_LENGTH_FOR_FULL_TEXT_SEARCH = 4;
protected $fullTextSearchDataCacheHash = [];
public function __construct($entityManager, \Espo\Entities\User $user, Acl $acl, AclManager $aclManager, Metadata $metadata, Config $config, InjectableFactory $injectableFactory)
{
$this->entityManager = $entityManager;
@@ -205,8 +207,8 @@ class Base
}
$this->applyBoolFilter($filter, $result);
}
} else if ($item['type'] == 'textFilter' && !empty($item['value'])) {
if (!empty($item['value'])) {
} else if ($item['type'] == 'textFilter') {
if (isset($item['value']) || $item['value'] !== '') {
$this->textFilter($item['value'], $result);
}
} else if ($item['type'] == 'primary' && !empty($item['value'])) {
@@ -389,7 +391,7 @@ class Base
protected function q($params, &$result)
{
if (!empty($params['q'])) {
if (isset($params['q']) && $params['q'] !== '') {
$this->textFilter($params['q'], $result);
}
}
@@ -403,7 +405,7 @@ class Base
public function manageTextFilter($textFilter, &$result)
{
$this->prepareResult($result);
$this->q(array('q' => $textFilter), $result);
$this->q(['q' => $textFilter], $result);
}
public function getEmptySelectParams()
@@ -734,7 +736,7 @@ class Base
$this->where($params['where'], $result);
}
if (!empty($params['textFilter'])) {
if (isset($params['textFilter']) && $params['textFilter'] !== '') {
$this->textFilter($params['textFilter'], $result);
}
@@ -1502,18 +1504,32 @@ class Base
);
}
public function getFullTextSearchDataForTextFilter($textFilter)
public function getFullTextSearchDataForTextFilter($textFilter, $stripWildcard = false)
{
if (array_key_exists($textFilter, $this->fullTextSearchDataCacheHash)) {
return $this->fullTextSearchDataCacheHash[$textFilter];
}
if ($this->getConfig()->get('fullTextSearchIsNotAvailable')) {
return null;
}
$result = null;
$fieldList = $this->getTextFilterFieldList();
$useFullTextSearch = false;
if ($this->getMetadata()->get(['scopes', $this->getEntityType(), 'fullTextSearch'])) {
if (mb_strlen($textFilter) >= $this->getConfig()->get('fullTextSearchMinLength', self::MIN_LENGTH_FOR_FULL_TEXT_SEARCH)) {
$useFullTextSearch = true;
if ($stripWildcard) {
$textFilter = str_replace('%', '', $textFilter);
}
if (mb_strpos($textFilter, '%')) {
$useFullTextSearch = false;
}
$useFullTextSearch = false;
if ($this->getMetadata()->get(['entityDefs', $this->getEntityType(), 'collection', 'fullTextSearch'])) {
$fullTextSearchMinLength = $this->getConfig()->get('fullTextSearchMinLength', self::MIN_LENGTH_FOR_FULL_TEXT_SEARCH);
if (!$fullTextSearchMinLength) {
$fullTextSearchMinLength = 0;
}
if (mb_strlen($textFilter) >= $fullTextSearchMinLength) {
$useFullTextSearch = true;
}
}
@@ -1553,14 +1569,16 @@ class Base
$fullTextSearchFieldSanitizedList[$i] = $this->getEntityManager()->getQuery()->sanitize($field);
}
$where = $function . ':' . implode(',', $fullTextSearchFieldSanitizedList) . ':' . $this->getEntityManager()->getQuery()->quote($textFilter);
return [
$where = $function . ':' . implode(',', $fullTextSearchFieldSanitizedList) . ':' . $textFilter;
$result = [
'where' => $where,
'fieldList' => $fullTextSearchFieldList
];
}
return null;
$this->fullTextSearchDataCacheHash[$textFilter] = $result;
return $result;
}
protected function textFilter($textFilter, &$result)
@@ -1571,40 +1589,91 @@ class Base
$textFilterContainsMinLength = $this->getConfig()->get('textFilterContainsMinLength', self::MIN_LENGTH_FOR_CONTENT_SEARCH);
$fullTextSearchData = $this->getFullTextSearchDataForTextFilter($textFilter);
$fullTextSearchData = null;
$forceFullTextSearch = false;
$useFullTextSearch = !empty($result['forceFullTextSearch']);
if (mb_strpos($textFilter, 'ft:') === 0) {
$textFilter = mb_substr($textFilter, 3);
$useFullTextSearch = true;
}
$skipWidlcards = false;
if (!$useFullTextSearch) {
if (mb_strpos($textFilter, '*') !== false) {
$skipWidlcards = true;
$textFilter = str_replace('*', '%', $textFilter);
}
}
$fullTextSearchData = $this->getFullTextSearchDataForTextFilter($textFilter, !$useFullTextSearch);
$fullTextGroup = [];
$fullTextSearchFieldList = [];
if ($fullTextSearchData) {
$group[] = $fullTextSearchData['where'];
$fullTextGroup[] = $fullTextSearchData['where'];
$fullTextSearchFieldList = $fullTextSearchData['fieldList'];
}
foreach ($fieldList as $field) {
if (in_array($field, $fullTextSearchFieldList)) continue;
if ($useFullTextSearch) {
if (in_array($field, $fullTextSearchFieldList)) continue;
}
if ($forceFullTextSearch) continue;
$attributeType = null;
if (!empty($fieldDefs[$field]['type'])) {
$attributeType = $fieldDefs[$field]['type'];
}
if (
strlen($textFilter) >= $textFilterContainsMinLength
&&
(
$attributeType == 'text'
||
!empty($this->textFilterUseContainsAttributeList[$field])
||
$attributeType == 'varchar' && $this->getConfig()->get('textFilterUseContainsForVarchar')
)
) {
$expression = '%' . $textFilter . '%';
if (!$skipWidlcards) {
if (
strlen($textFilter) >= $textFilterContainsMinLength
&&
(
$attributeType == 'text'
||
!empty($this->textFilterUseContainsAttributeList[$field])
||
$attributeType == 'varchar' && $this->getConfig()->get('textFilterUseContainsForVarchar')
)
) {
$expression = '%' . $textFilter . '%';
} else {
$expression = $textFilter . '%';
}
} else {
$expression = $textFilter . '%';
$expression = $textFilter;
}
if ($fullTextSearchData) {
if (!$useFullTextSearch) {
if (in_array($field, $fullTextSearchFieldList)) {
if (!array_key_exists('OR', $fullTextGroup)) {
$fullTextGroup['OR'] = [];
}
$fullTextGroup['OR'][$field . '*'] = $expression;
continue;
}
}
}
$group[$field . '*'] = $expression;
}
if (!empty($fullTextGroup)) {
$group['AND'] = $fullTextGroup;
}
if (count($group) === 0) {
$result['whereClause'][] = [
'id' => null
];
}
$result['whereClause'][] = [
'OR' => $group
];
application/Espo/ORM/DB/Query/Base.php
+3 -1
View File
@@ -340,8 +340,10 @@ abstract class Base
$columnList = explode(',', $columns);
$tableName = $this->toDb($entity->getEntityType());
foreach ($columnList as $i => $column) {
$columnList[$i] = $this->sanitize($column);
$columnList[$i] = $tableName . '.' . $this->sanitize($column);
}
$query = $this->quote($query);
application/Espo/Services/GlobalSearch.php
+4 -3
View File
@@ -82,7 +82,7 @@ class GlobalSearch extends \Espo\Core\Services\Base
if (!$this->getAcl()->checkScope($entityType, 'read')) {
continue;
}
if (!$this->getMetadata()->get('scopes.' . $entityType)) {
if (!$this->getMetadata()->get(['scopes', $entityType])) {
continue;
}
@@ -98,12 +98,13 @@ class GlobalSearch extends \Espo\Core\Services\Base
$hasFullTextSearch = true;
$params['select'][] = [$fullTextSearchData['where'], '_relevance'];
} else {
$params['select'][] = ['VALUE:0.9', '_relevance'];
$params['select'][] = ['VALUE:1.1', '_relevance'];
$relevanceSelectPosition = count($params['select']);
}
$selectManager->manageAccess($params);
$selectManager->manageTextFilter($query, $params);
$params['forceFullTextSearch'] = true;
$selectManager->applyTextFilter($query, $params);
$sql = $this->getEntityManager()->getQuery()->createSelectQuery($entityType, $params);
tests/unit/Espo/ORM/DB/QueryTest.php
+22 -6
View File
@@ -455,7 +455,7 @@ class QueryTest extends \PHPUnit\Framework\TestCase
$expectedSql =
"SELECT article.id AS `id`, article.name AS `name` FROM `article` " .
"WHERE MATCH (name,description) AGAINST ('test +hello' IN BOOLEAN MODE) AND article.id IS NOT NULL AND article.deleted = '0'";
"WHERE MATCH (article.name,article.description) AGAINST ('test +hello' IN BOOLEAN MODE) AND article.id IS NOT NULL AND article.deleted = '0'";
$this->assertEquals($expectedSql, $sql);
}
@@ -471,7 +471,7 @@ class QueryTest extends \PHPUnit\Framework\TestCase
$expectedSql =
"SELECT article.id AS `id`, article.name AS `name` FROM `article` " .
"WHERE MATCH (description) AGAINST ('\"test hello\"' IN NATURAL LANGUAGE MODE) AND article.deleted = '0'";
"WHERE MATCH (article.description) AGAINST ('\"test hello\"' IN NATURAL LANGUAGE MODE) AND article.deleted = '0'";
$this->assertEquals($expectedSql, $sql);
}
@@ -489,8 +489,8 @@ class QueryTest extends \PHPUnit\Framework\TestCase
]);
$expectedSql =
"SELECT article.id AS `id`, MATCH (description) AGAINST ('test' IN BOOLEAN MODE) AS `MATCH_BOOLEAN:description:test` FROM `article` " .
"WHERE MATCH (description) AGAINST ('test' IN BOOLEAN MODE) AND article.deleted = '0' " .
"SELECT article.id AS `id`, MATCH (article.description) AGAINST ('test' IN BOOLEAN MODE) AS `MATCH_BOOLEAN:description:test` FROM `article` " .
"WHERE MATCH (article.description) AGAINST ('test' IN BOOLEAN MODE) AND article.deleted = '0' " .
"ORDER BY 2 DESC";
$this->assertEquals($expectedSql, $sql);
@@ -509,10 +509,26 @@ class QueryTest extends \PHPUnit\Framework\TestCase
]);
$expectedSql =
"SELECT article.id AS `id`, MATCH (description) AGAINST ('test' IN BOOLEAN MODE) AS `relevance` FROM `article` " .
"WHERE MATCH (description) AGAINST ('test' IN BOOLEAN MODE) AND article.deleted = '0' " .
"SELECT article.id AS `id`, MATCH (article.description) AGAINST ('test' IN BOOLEAN MODE) AS `relevance` FROM `article` " .
"WHERE MATCH (article.description) AGAINST ('test' IN BOOLEAN MODE) AND article.deleted = '0' " .
"ORDER BY 2 DESC";
$this->assertEquals($expectedSql, $sql);
}
public function testMatch5()
{
$sql = $this->query->createSelectQuery('Article', [
'select' => ['id', 'name'],
'whereClause' => [
'MATCH_NATURAL_LANGUAGE:description:test>' => 1
]
]);
$expectedSql =
"SELECT article.id AS `id`, article.name AS `name` FROM `article` " .
"WHERE MATCH (article.description) AGAINST ('test' IN NATURAL LANGUAGE MODE) > '1' AND article.deleted = '0'";
$this->assertEquals($expectedSql, $sql);
}
}