transform favorite, tag and systemtag search queries to fileid lists

not as performant but compatible with sharding

Signed-off-by: Robin Appelman <[email protected]>

lib/private/Files/Cache/QuerySearchHelper.php

@@ -24,11 +24,17 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  */
+
 namespace OC\Files\Cache;
 
 use OC\Files\Cache\Wrapper\CacheJail;
+use OC\Files\Search\QueryOptimizer\DavTagToFileIds;
+use OC\Files\Search\QueryOptimizer\FavoriteToTag;
 use OC\Files\Search\QueryOptimizer\QueryOptimizer;
+use OC\Files\Search\QueryOptimizer\SystemTagToFileIds;
 use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery;
 use OC\SystemConfig;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Cache\ICache;
@@ -47,13 +53,13 @@
 
 class QuerySearchHelper {
  public function __construct(
- private IMimeTypeLoader $mimetypeLoader,
- private IDBConnection $connection,
- private SystemConfig $systemConfig,
- private LoggerInterface $logger,
- private SearchBuilder $searchBuilder,
- private QueryOptimizer $queryOptimizer,
- private IGroupManager $groupManager,
+ private IMimeTypeLoader  $mimetypeLoader,
+ private IDBConnection  $connection,
+ private SystemConfig  $systemConfig,
+ private LoggerInterface  $logger,
+ private SearchBuilder  $searchBuilder,
+ private QueryOptimizer  $queryOptimizer,
+ private IGroupManager  $groupManager,
  private IFilesMetadataManager $filesMetadataManager,
  private CacheDatabase $cacheDatabase,
  ) {
@@ -76,9 +82,9 @@ protected function getQueryBuilder() {
  */
  protected function applySearchConstraints(
  CacheQueryBuilder $query,
- ISearchQuery $searchQuery,
- array $caches,
- ?IMetadataQuery $metadataQuery = null
+ ISearchQuery  $searchQuery,
+ array  $caches,
+ ?IMetadataQuery  $metadataQuery = null
  ): void {
  $storageFilters = array_values(array_map(function (ICache $cache) {
  return $cache->getQueryFilterForStorage();
@@ -132,18 +138,6 @@ protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user
  $query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on);
  }
 
- protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void {
- $query
- ->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid'))
- ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX(
- $query->expr()->eq('tagmap.type', 'tag.type'),
- $query->expr()->eq('tagmap.categoryid', 'tag.id'),
- $query->expr()->eq('tag.type', $query->createNamedParameter('files')),
- $query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID()))
- ));
- }
-
-
  protected function equipQueryForShares(CacheQueryBuilder $query): void {
  $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source'));
  }
@@ -174,20 +168,32 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array
  // while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id)
  // we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result.
 
- $storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches);
- $cachesByStorage = array_combine($storageIds, $caches);
+ $operation = $searchQuery->getSearchOperation();
 
  $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation());
 
+ if (in_array('favorite', $requestedFields)) {
+ $favoriteHandler = new FavoriteToTag();
+ $favoriteHandler->processOperator($operation);
+ }
+
+ if (in_array('systemtag', $requestedFields)) {
+ $systemTagHandler = new SystemTagToFileIds($this->connection, $this->requireUser($searchQuery), $this->groupManager);
+ $systemTagHandler->processOperator($operation);
+ }
+ if (in_array('tagname', $requestedFields)) {
+ $davTagHandler = new DavTagToFileIds($this->connection, $this->requireUser($searchQuery));
+ $davTagHandler->processOperator($operation);
+ }
+
+ $searchQuery->setSearchOperation($operation);
+
+ $storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches);
+ $cachesByStorage = array_combine($storageIds, $caches);
+
  $rawEntries = $this->cacheDatabase->queryStorages($storageIds, function(CacheQueryBuilder $builder, $storages) use ($requestedFields, $searchQuery, $cachesByStorage) {
  $cachesForShard = array_map(fn (int $storage) => $cachesByStorage[$storage], $storages);
  $query = $builder->selectFileCache('file', false);
- if (in_array('systemtag', $requestedFields)) {
- $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery));
- }
- if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) {
- $this->equipQueryForDavTags($query, $this->requireUser($searchQuery));
- }
  if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) {
  $this->equipQueryForShares($query);
  }

lib/private/Files/Cache/SearchBuilder.php

@@ -75,9 +75,6 @@ class SearchBuilder {
  'name' => 'string',
  'path' => 'string',
  'size' => 'integer',
- 'tagname' => 'string',
- 'systemtag' => 'string',
- 'favorite' => 'boolean',
  'fileid' => 'integer',
  'storage' => 'integer',
  'share_with' => 'string',
@@ -99,8 +96,6 @@ class SearchBuilder {
  'boolean' => IQueryBuilder::PARAM_INT_ARRAY,
  ];
 
- public const TAG_FAVORITE = '_$!<Favorite>!$_';
-
  /** @var IMimeTypeLoader */
  private $mimetypeLoader;
 
@@ -243,16 +238,8 @@ private function getOperatorFieldAndValueInner(string $field, mixed $value, stri
  $type = ISearchComparison::COMPARE_EQUAL;
  }
  }
- } elseif ($field === 'favorite') {
- $field = 'tag.category';
- $value = self::TAG_FAVORITE;
- $paramType = 'string';
  } elseif ($field === 'name') {
  $field = 'file.name';
- } elseif ($field === 'tagname') {
- $field = 'tag.category';
- } elseif ($field === 'systemtag') {
- $field = 'systemtag.name';
  } elseif ($field === 'fileid') {
  $field = 'file.fileid';
  } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) {
@@ -271,9 +258,6 @@ private function validateComparison(ISearchComparison $operator) {
  'name' => ['eq', 'like', 'clike', 'in'],
  'path' => ['eq', 'like', 'clike', 'in'],
  'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
- 'tagname' => ['eq', 'like'],
- 'systemtag' => ['eq', 'like'],
- 'favorite' => ['eq'],
  'fileid' => ['eq', 'in'],
  'storage' => ['eq', 'in'],
  'share_with' => ['eq'],

lib/private/Files/Search/QueryOptimizer/DavTagToFileIds.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2024 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+use OCP\IDBConnection;
+use OCP\IGroupManager;
+use OCP\IUser;
+
+class DavTagToFileIds extends ReplacingOptimizerStep {
+ public function __construct(
+ private IDBConnection $connection,
+ private IUser $user,
+ ) {
+ }
+
+ public function processOperator(ISearchOperator &$operator): bool {
+ if ($operator instanceof ISearchComparison && $operator->getField() === 'tagname') {
+ $operator = new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $this->getFileIdsForDavTag($operator));
+ return true;
+ } else {
+ return parent::processOperator($operator);
+ }
+ }
+
+
+ private function getFileIdsForDavTag(ISearchComparison $comparison): array {
+ $query = $this->connection->getQueryBuilder();
+
+ $query->select('tagmap.objid')
+ ->from('vcategory_to_object', 'tagmap')
+ ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX(
+ $query->expr()->eq('tagmap.type', 'tag.type'),
+ $query->expr()->eq('tagmap.categoryid', 'tag.id')
+ ))
+ ->where($query->expr()->eq('tag.type', $query->createNamedParameter('files')))
+ ->andWhere($query->expr()->eq('tag.uid', $query->createNamedParameter($this->user->getUID())));
+ if ($comparison->getType() === ISearchComparison::COMPARE_EQUAL) {
+ $query->andWhere($query->expr()->eq('tag.category', $query->createNamedParameter($comparison->getValue())));
+ } elseif ($comparison->getType() === ISearchComparison::COMPARE_LIKE) {
+ $query->andWhere($query->expr()->like('tag.category', $query->createNamedParameter($comparison->getValue())));
+ } else {
+ throw new \InvalidArgumentException('Unsupported comparison for field ' . $comparison->getField() . ': ' . $comparison->getType());
+ }
+ return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ }
+}

lib/private/Files/Search/QueryOptimizer/FavoriteToTag.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2024 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+use OCP\IDBConnection;
+use OCP\IGroupManager;
+use OCP\IUser;
+
+class FavoriteToTag extends ReplacingOptimizerStep {
+ public const TAG_FAVORITE = '_$!<Favorite>!$_';
+
+ public function processOperator(ISearchOperator &$operator): bool {
+ if ($operator instanceof ISearchComparison && $operator->getField() === 'favorite' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) {
+ $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'tagname', self::TAG_FAVORITE);
+ return true;
+ } else {
+ return parent::processOperator($operator);
+ }
+ }
+}

lib/private/Files/Search/QueryOptimizer/SystemTagToFileIds.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2024 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+use OCP\IDBConnection;
+use OCP\IGroupManager;
+use OCP\IUser;
+
+class SystemTagToFileIds extends ReplacingOptimizerStep {
+ public function __construct(
+ private IDBConnection $connection,
+ private IUser $user,
+ private IGroupManager $groupManager,
+ ) {
+ }
+
+ public function processOperator(ISearchOperator &$operator): bool {
+ if ($operator instanceof ISearchComparison && $operator->getField() === 'systemtag') {
+ $operator = new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $this->getFileIdsForSystemTag($operator));
+ return true;
+ } else {
+ return parent::processOperator($operator);
+ }
+ }
+
+
+ private function getFileIdsForSystemTag(ISearchComparison $comparison): array {
+ $query = $this->connection->getQueryBuilder();
+
+ $on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'));
+ if (!$this->groupManager->isAdmin($this->user->getUID())) {
+ $on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true)));
+ }
+
+ $query->select('systemtagmap.objectid')
+ ->from('systemtag_object_mapping', 'systemtagmap')
+ ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on)
+ ->where($query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')))
+ ->andWhere($query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')));
+ if ($comparison->getType() === ISearchComparison::COMPARE_EQUAL) {
+ $query->andWhere($query->expr()->eq('systemtag.name', $query->createNamedParameter($comparison->getValue())));
+ } elseif ($comparison->getType() === ISearchComparison::COMPARE_LIKE) {
+ $query->andWhere($query->expr()->like('systemtag.name', $query->createNamedParameter($comparison->getValue())));
+ } else {
+ throw new \InvalidArgumentException('Unsupported comparison for field ' . $comparison->getField() . ': ' . $comparison->getType());
+ }
+ return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ }
+}

lib/private/Files/Search/SearchQuery.php

@@ -73,6 +73,10 @@ public function getSearchOperation() {
  return $this->searchOperation;
  }
 
+ public function setSearchOperation(ISearchOperator $operator): void {
+ $this->searchOperation = $operator;
+ }
+
  /**
  * @return int
  */

lib/public/Files/Search/ISearchQuery.php

@@ -34,6 +34,13 @@ interface ISearchQuery {
  */
  public function getSearchOperation();
 
+ /**
+ * @param ISearchOperator $operator
+ * @return void
+ * @since 29.0.0
+ */
+ public function setSearchOperation(ISearchOperator $operator): void;
+
  /**
  * Get the maximum number of results to return
  *