From 216a92edf65c11d6a63cf28f675cec4d3e87f45a Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 14 Feb 2024 16:05:34 +0100 Subject: [PATCH 1/7] feat: add class for (future) shared access to the filecache Signed-off-by: Robin Appelman --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Files/Cache/Cache.php | 67 +++----- lib/private/Files/Cache/CacheDatabase.php | 157 ++++++++++++++++++ lib/private/Files/Cache/CacheDependencies.php | 19 ++- lib/private/Files/Cache/CacheQueryBuilder.php | 12 +- lib/private/Files/Cache/Storage.php | 2 +- .../Files/Cache/Wrapper/CacheWrapper.php | 8 +- tests/lib/Files/Cache/CacheTest.php | 20 --- 9 files changed, 203 insertions(+), 84 deletions(-) create mode 100644 lib/private/Files/Cache/CacheDatabase.php diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 8e1408e121ef6..190538ee03e59 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1405,6 +1405,7 @@ 'OC\\Files\\Cache\\CacheDependencies' => $baseDir . '/lib/private/Files/Cache/CacheDependencies.php', 'OC\\Files\\Cache\\CacheEntry' => $baseDir . '/lib/private/Files/Cache/CacheEntry.php', 'OC\\Files\\Cache\\CacheQueryBuilder' => $baseDir . '/lib/private/Files/Cache/CacheQueryBuilder.php', + 'OC\\Files\\Cache\\Database' => $baseDir . '/lib/private/Files/Cache/Database.php', 'OC\\Files\\Cache\\FailedCache' => $baseDir . '/lib/private/Files/Cache/FailedCache.php', 'OC\\Files\\Cache\\FileAccess' => $baseDir . '/lib/private/Files/Cache/FileAccess.php', 'OC\\Files\\Cache\\HomeCache' => $baseDir . '/lib/private/Files/Cache/HomeCache.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index d6939ae36ce8b..90b669fcd2c5d 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1446,6 +1446,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\Cache\\CacheDependencies' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheDependencies.php', 'OC\\Files\\Cache\\CacheEntry' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheEntry.php', 'OC\\Files\\Cache\\CacheQueryBuilder' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheQueryBuilder.php', + 'OC\\Files\\Cache\\Database' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Database.php', 'OC\\Files\\Cache\\FailedCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FailedCache.php', 'OC\\Files\\Cache\\FileAccess' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FileAccess.php', 'OC\\Files\\Cache\\HomeCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/HomeCache.php', diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 5267b8a815392..92300a080f9c4 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -62,7 +62,7 @@ use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; use OCP\FilesMetadata\IFilesMetadataManager; -use OCP\IDBConnection; +use OCP\Server; use OCP\Util; use Psr\Log\LoggerInterface; @@ -88,7 +88,7 @@ class Cache implements ICache { protected string $storageId; protected Storage $storageCache; protected IMimeTypeLoader$mimetypeLoader; - protected IDBConnection $connection; + protected CacheDatabase $cacheDb; protected SystemConfig $systemConfig; protected LoggerInterface $logger; protected QuerySearchHelper $querySearchHelper; @@ -106,11 +106,11 @@ public function __construct( $this->storageId = md5($this->storageId); } if (!$dependencies) { - $dependencies = \OC::$server->get(CacheDependencies::class); + $dependencies = Server::get(CacheDependencies::class); } $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); - $this->connection = $dependencies->getConnection(); + $this->cacheDb = $dependencies->getCacheDb(); $this->systemConfig = $dependencies->getSystemConfig(); $this->logger = $dependencies->getLogger(); $this->querySearchHelper = $dependencies->getQuerySearchHelper(); @@ -118,13 +118,8 @@ public function __construct( $this->metadataManager = $dependencies->getMetadataManager(); } - protected function getQueryBuilder() { - return new CacheQueryBuilder( - $this->connection, - $this->systemConfig, - $this->logger, - $this->metadataManager, - ); + public function getQueryBuilder() { + return $this->cacheDb->queryForStorageId($this->getNumericStorageId()); } public function getStorageCache(): Storage { @@ -305,7 +300,7 @@ public function insert($file, array $data) { $values['storage'] = $storageId; try { - $builder = $this->connection->getQueryBuilder(); + $builder = $this->getQueryBuilder(); $builder->insert('filecache'); foreach ($values as $column => $value) { @@ -333,9 +328,9 @@ public function insert($file, array $data) { } } catch (UniqueConstraintViolationException $e) { // entry exists already - if ($this->connection->inTransaction()) { - $this->connection->commit(); - $this->connection->beginTransaction(); + if ($this->cacheDb->inTransaction($this->getNumericStorageId())) { + $this->cacheDb->commit($this->getNumericStorageId()); + $this->cacheDb->beginTransaction($this->getNumericStorageId()); } } @@ -699,7 +694,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { if ($sourceData['mimetype'] === 'httpd/unix-directory') { //update all child entries $sourceLength = mb_strlen($sourcePath); - $query = $this->connection->getQueryBuilder(); + $query = $this->getQueryBuilder(); $fun = $query->func(); $newPathFunction = $fun->concat( @@ -711,7 +706,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { ->set('path_hash', $fun->md5($newPathFunction)) ->set('path', $newPathFunction) ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%'))); + ->andWhere($query->expr()->like('path', $query->createNamedParameter($query->escapeLikeParameter($sourcePath) . '/%'))); // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { @@ -723,11 +718,11 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { $retryLimit = 4; for ($i = 1; $i <= $retryLimit; $i++) { try { - $this->connection->beginTransaction(); + $this->cacheDb->beginTransaction(); $query->executeStatement(); break; } catch (\OC\DatabaseException $e) { - $this->connection->rollBack(); + $this->cacheDb->rollBack($this->getNumericStorageId()); throw $e; } catch (RetryableException $e) { // Simply throw if we already retried 4 times. @@ -735,14 +730,14 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { throw $e; } - $this->connection->rollBack(); + $this->cacheDb->rollBack(); // Sleep a bit to give some time to the other transaction to finish. usleep(100 * 1000 * $i); } } } else { - $this->connection->beginTransaction(); + $this->cacheDb->beginTransaction(); } $query = $this->getQueryBuilder(); @@ -761,7 +756,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { $query->execute(); - $this->connection->commit(); + $this->cacheDb->commit($this->getNumericStorageId()); if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) { $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId())); @@ -787,7 +782,7 @@ public function clear() { ->whereStorageId($this->getNumericStorageId()); $query->execute(); - $query = $this->connection->getQueryBuilder(); + $query = $this->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); $query->execute(); @@ -861,8 +856,8 @@ public function searchByMime($mimetype) { return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** @@ -1099,27 +1094,7 @@ public function getPathById($id) { * @deprecated use getPathById() instead */ public static function getById($id) { - $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); - $query->select('path', 'storage') - ->from('filecache') - ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); - - $result = $query->execute(); - $row = $result->fetch(); - $result->closeCursor(); - - if ($row) { - $numericId = $row['storage']; - $path = $row['path']; - } else { - return null; - } - - if ($id = Storage::getStorageId($numericId)) { - return [$id, $path]; - } else { - return null; - } + throw new \Exception("Cache::getById has been removed"); } /** diff --git a/lib/private/Files/Cache/CacheDatabase.php b/lib/private/Files/Cache/CacheDatabase.php new file mode 100644 index 0000000000000..4e7a48cc45c9c --- /dev/null +++ b/lib/private/Files/Cache/CacheDatabase.php @@ -0,0 +1,157 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Cache; + +use OC\DB\Exceptions\DbalException; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\ICache; +use OCP\ICacheFactory; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class CacheDatabase { + private ICache $cache; + + public function __construct( + private IDBConnection $connection, // todo: multiple db connections for sharding (open connection lazy?) + private SystemConfig $systemConfig, + private LoggerInterface $logger, + ICacheFactory $cacheFactory, + ) { + $this->cache = $cacheFactory->createLocal('storage_by_fileid'); + } + + private function connectionForStorageId(int $storage): IDBConnection { + return $this->databaseForShard($this->getShardForStorageId($storage)); + } + + public function queryForStorageId(int $storage): CacheQueryBuilder { + return $this->queryForShard($this->getShardForStorageId($storage)); + } + + private function databaseForShard(int $shard): IDBConnection { + return $this->connection; + } + + public function queryForShard(int $shard): CacheQueryBuilder { + // todo: select db based on shard + $query = new CacheQueryBuilder( + $this->databaseForShard($shard), + $this->systemConfig, + $this->logger, + ); + $query->setSharded(true); + return $query; + } + + public function getCachedStorageIdForFileId(int $fileId): ?int { + $cached = $this->cache->get((string)$fileId); + return ($cached === null) ? null : (int)$cached; + } + + public function setCachedStorageIdForFileId(int $fileId, int $storageId) { + $this->cache->set((string)$fileId, $storageId); + } + + /** + * @param list $fileIds + * @return array> + */ + public function getCachedShardsForFileIds(array $fileIds): array { + $result = []; + foreach ($fileIds as $fileId) { + $storageId = $this->getCachedStorageIdForFileId($fileId); + if ($storageId) { + $shard = $this->getShardForStorageId($storageId); + $result[$shard][] = $fileId; + } + } + return $result; + } + + private function getShardForStorageId(int $storage): int { + return 0; + } + + /** + * @return list + */ + public function getAllShards(): array { + return [0]; + } + + public function beginTransaction(int $storageId): void { + $this->connectionForStorageId($storageId)->beginTransaction(); + } + + public function inTransaction(int $storageId): bool { + return $this->connectionForStorageId($storageId)->inTransaction(); + } + + public function commit(int $storageId): void { + $this->connectionForStorageId($storageId)->commit(); + } + + public function rollBack(int $storageId): void { + $this->connectionForStorageId($storageId)->rollBack(); + } + + /** + * @param List $storages + * @return array> + */ + private function groupStoragesByShard(array $storages): array { + $storagesByShard = []; + foreach ($storages as $storage) { + $shard = $this->getShardForStorageId($storage); + $storagesByShard[$shard][] = $storage; + } + return $storagesByShard; + } + + /** + * Run a query against all shards for the given storage ids, combining the results. + * + * The provided callback fill be called with the query builder for each shard and the storage ids for that shard. + * The results from the callback will be combined and returned + * + * @template T + * @param List $storages + * @param callable(CacheQueryBuilder, List): T[] $callback + * @return T[] + */ + public function queryStorages(array $storages, callable $callback): array { + $result = []; + + $storagesByShard = $this->groupStoragesByShard($storages); + foreach($storagesByShard as $shard => $storagesForShard) { + $query = $this->queryForShard($shard); + $shardResults = $callback($query, $storagesForShard); + $result += $shardResults; + } + return $result; + } +} diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php index 7c51f3ff88421..2a84ce7357433 100644 --- a/lib/private/Files/Cache/CacheDependencies.php +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -12,14 +12,15 @@ class CacheDependencies { public function __construct( - private IMimeTypeLoader $mimeTypeLoader, - private IDBConnection $connection, - private IEventDispatcher $eventDispatcher, - private QuerySearchHelper $querySearchHelper, - private SystemConfig $systemConfig, - private LoggerInterface $logger, + private IMimeTypeLoader $mimeTypeLoader, + private IDBConnection $connection, + private IEventDispatcher $eventDispatcher, + private QuerySearchHelper $querySearchHelper, + private SystemConfig $systemConfig, + private LoggerInterface $logger, private IFilesMetadataManager $metadataManager, - private DisplayNameCache $displayNameCache, + private DisplayNameCache $displayNameCache, + private CacheDatabase $cacheDb, ) { } @@ -54,4 +55,8 @@ public function getDisplayNameCache(): DisplayNameCache { public function getMetadataManager(): IFilesMetadataManager { return $this->metadataManager; } + + public function getCacheDb(): CacheDatabase { + return $this->cacheDb; + } } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index a237dd915d735..4741787198e7b 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -68,7 +68,7 @@ public function selectTagUsage(): self { return $this; } - public function selectFileCache(?string $alias = null, bool $joinExtendedCache = true) { + public function selectFileCache(?string $alias = null, bool $joinExtendedCache = true): self { $name = $alias ?: 'filecache'; $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', 'storage_mtime', 'encrypted', 'etag', "$name.permissions", 'checksum', 'unencrypted_size') @@ -84,13 +84,13 @@ public function selectFileCache(?string $alias = null, bool $joinExtendedCache = return $this; } - public function whereStorageId(int $storageId) { + public function whereStorageId(int $storageId): self { $this->andWhere($this->expr()->eq('storage', $this->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); return $this; } - public function whereFileId(int $fileId) { + public function whereFileId(int $fileId): self { $alias = $this->alias; if ($alias) { $alias .= '.'; @@ -103,13 +103,13 @@ public function whereFileId(int $fileId) { return $this; } - public function wherePath(string $path) { + public function wherePath(string $path): self { $this->andWhere($this->expr()->eq('path_hash', $this->createNamedParameter(md5($path)))); return $this; } - public function whereParent(int $parent) { + public function whereParent(int $parent): self { $alias = $this->alias; if ($alias) { $alias .= '.'; @@ -122,7 +122,7 @@ public function whereParent(int $parent) { return $this; } - public function whereParentInParameter(string $parameter) { + public function whereParentInParameter(string $parameter): self { $alias = $this->alias; if ($alias) { $alias .= '.'; diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index ba0f98f42f492..af7f7e19c10e0 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -62,7 +62,7 @@ public static function getGlobalCache() { } /** - * @param \OC\Files\Storage\Storage|string $storage + * @param IStorage|string $storage * @param bool $isAvailable * @throws \RuntimeException */ diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index 1662c76a6b8b1..6d7c37379706e 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -47,14 +47,14 @@ public function __construct(?ICache $cache, ?CacheDependencies $dependencies = n $this->cache = $cache; if (!$dependencies && $cache instanceof Cache) { $this->mimetypeLoader = $cache->mimetypeLoader; - $this->connection = $cache->connection; + $this->cacheDb = $cache->cacheDb; $this->querySearchHelper = $cache->querySearchHelper; } else { if (!$dependencies) { $dependencies = Server::get(CacheDependencies::class); } $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); - $this->connection = $dependencies->getConnection(); + $this->cacheDb = $dependencies->getCacheDb(); $this->querySearchHelper = $dependencies->getQuerySearchHelper(); } } @@ -236,8 +236,8 @@ public function getStatus($file) { return $this->getCache()->getStatus($file); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** diff --git a/tests/lib/Files/Cache/CacheTest.php b/tests/lib/Files/Cache/CacheTest.php index faecbf54491c0..cd648fba1c3d1 100644 --- a/tests/lib/Files/Cache/CacheTest.php +++ b/tests/lib/Files/Cache/CacheTest.php @@ -506,17 +506,6 @@ public function testNonExisting() { $this->assertEquals([], $this->cache->getFolderContents('foo')); } - public function testGetById() { - $storageId = $this->storage->getId(); - $data = ['size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file']; - $id = $this->cache->put('foo', $data); - - if (strlen($storageId) > 64) { - $storageId = md5($storageId); - } - $this->assertEquals([$storageId, 'foo'], \OC\Files\Cache\Cache::getById($id)); - } - public function testStorageMTime() { $data = ['size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file']; $this->cache->put('foo', $data); @@ -534,15 +523,6 @@ public function testStorageMTime() { $this->assertEquals(25, $cachedData['mtime']); } - public function testLongId() { - $storage = new LongId([]); - $cache = $storage->getCache(); - $storageId = $storage->getId(); - $data = ['size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file']; - $id = $cache->put('foo', $data); - $this->assertEquals([md5($storageId), 'foo'], \OC\Files\Cache\Cache::getById($id)); - } - /** * this test show the bug resulting if we have no normalizer installed */ From 647e64dec03a30762f99d4492fd77be8cf3cf4bd Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 14 Feb 2024 16:04:35 +0100 Subject: [PATCH 2/7] expose escapeLikeParameter trough query builder Signed-off-by: Robin Appelman --- lib/private/DB/QueryBuilder/QueryBuilder.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 3fc56dd580db1..5fb9825a6fdc9 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -1337,4 +1337,8 @@ public function quoteAlias($alias) { return $this->helper->quoteColumnName($alias); } + + public function escapeLikeParameter(string $parameter): string { + return $this->connection->escapeLikeParameter($parameter); + } } From b3b0c14f5d6f09f757ed8ef5ffabb9b8f8c9aa1a Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 14 Feb 2024 16:05:13 +0100 Subject: [PATCH 3/7] block access to the filecache tables by default Signed-off-by: Robin Appelman --- lib/private/DB/QueryBuilder/QueryBuilder.php | 45 ++++++++++++++++++++ tests/lib/Files/Cache/SearchBuilderTest.php | 3 ++ 2 files changed, 48 insertions(+) diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 5fb9825a6fdc9..3982b40c193d9 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -76,6 +76,19 @@ class QueryBuilder implements IQueryBuilder { /** @var string */ protected $lastInsertedTable; + /** + * Tables that require special attention and thus can't be queried by default + * + * @var list + */ + protected array $shardedTables = [ + 'filecache', + 'filecache_extended', + 'files_metadata' + ]; + + protected bool $sharded = false; + /** * Initializes a new QueryBuilder. * @@ -662,6 +675,17 @@ public function insert($insert = null) { return $this; } + /** + * @param string $table + * @return void + * @throws \Exception + */ + private function checkTableAccess(string $table) { + if (in_array($table, $this->shardedTables) !== $this->sharded) { + throw new \Exception("current query isn't allowed to access the $table table"); + } + } + /** * Creates and adds a query root corresponding to the table identified by the * given alias, forming a cartesian product with any existing query roots. @@ -678,6 +702,7 @@ public function insert($insert = null) { * @return $this This QueryBuilder instance. */ public function from($from, $alias = null) { + $this->checkTableAccess($from); $this->queryBuilder->from( $this->getTableName($from), $this->quoteAlias($alias) @@ -704,6 +729,8 @@ public function from($from, $alias = null) { * @return $this This QueryBuilder instance. */ public function join($fromAlias, $join, $alias, $condition = null) { + $this->checkTableAccess($join); + $this->queryBuilder->join( $this->quoteAlias($fromAlias), $this->getTableName($join), @@ -732,6 +759,8 @@ public function join($fromAlias, $join, $alias, $condition = null) { * @return $this This QueryBuilder instance. */ public function innerJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkTableAccess($join); + $this->queryBuilder->innerJoin( $this->quoteAlias($fromAlias), $this->getTableName($join), @@ -760,6 +789,8 @@ public function innerJoin($fromAlias, $join, $alias, $condition = null) { * @return $this This QueryBuilder instance. */ public function leftJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkTableAccess($join); + $this->queryBuilder->leftJoin( $this->quoteAlias($fromAlias), $this->getTableName($join), @@ -788,6 +819,8 @@ public function leftJoin($fromAlias, $join, $alias, $condition = null) { * @return $this This QueryBuilder instance. */ public function rightJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkTableAccess($join); + $this->queryBuilder->rightJoin( $this->quoteAlias($fromAlias), $this->getTableName($join), @@ -1341,4 +1374,16 @@ public function quoteAlias($alias) { public function escapeLikeParameter(string $parameter): string { return $this->connection->escapeLikeParameter($parameter); } + + /** + * Mark the query as accessing the sharded tables + * + * Proper attention needs to be given to ensure that all requirements for accessing the sharded tables + * + * @param bool $sharded + * @return void + */ + public function setSharded(bool $sharded): void { + $this->sharded = $sharded; + } } diff --git a/tests/lib/Files/Cache/SearchBuilderTest.php b/tests/lib/Files/Cache/SearchBuilderTest.php index 57d890fcbcd25..84cd4e2c694b2 100644 --- a/tests/lib/Files/Cache/SearchBuilderTest.php +++ b/tests/lib/Files/Cache/SearchBuilderTest.php @@ -7,6 +7,7 @@ namespace Test\Files\Cache; use OC\DB\QueryBuilder\Literal; +use OC\DB\QueryBuilder\QueryBuilder; use OC\Files\Cache\SearchBuilder; use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; @@ -35,7 +36,9 @@ class SearchBuilderTest extends TestCase { protected function setUp(): void { parent::setUp(); + /** @var QueryBuilder builder */ $this->builder = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $this->builder->setSharded(true); $this->mimetypeLoader = $this->createMock(IMimeTypeLoader::class); $this->mimetypeLoader->expects($this->any()) From 16112323eb1427522659661cebf4a807fc3cbf19 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 20 Feb 2024 14:30:25 +0100 Subject: [PATCH 4/7] adjust share provider wip Signed-off-by: Robin Appelman --- lib/private/Files/Cache/Cache.php | 4 +- lib/private/Files/Cache/CacheQueryBuilder.php | 5 +- lib/private/Files/Cache/QuerySearchHelper.php | 64 ++++++----- .../FilesMetadata/FilesMetadataManager.php | 4 +- .../Service/MetadataRequestService.php | 83 +++++++++----- lib/private/Share20/DefaultShareProvider.php | 107 +++++++----------- lib/private/Share20/ProviderFactory.php | 12 +- .../FilesMetadata/IFilesMetadataManager.php | 3 +- 8 files changed, 142 insertions(+), 140 deletions(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 92300a080f9c4..f3c66ae81bf5b 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -144,7 +144,7 @@ public function getNumericStorageId() { public function get($file) { $query = $this->getQueryBuilder(); $query->selectFileCache(); - $metadataQuery = $query->selectMetadata(); + $metadataQuery = $query->selectMetadata($this->metadataManager); if (is_string($file) || $file == '') { // normalize file @@ -231,7 +231,7 @@ public function getFolderContentsById($fileId) { ->whereParent($fileId) ->orderBy('name', 'ASC'); - $metadataQuery = $query->selectMetadata(); + $metadataQuery = $query->selectMetadata($this->metadataManager); $result = $query->execute(); $files = $result->fetchAll(); diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 4741787198e7b..3cd41bc2f0f5b 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -44,7 +44,6 @@ public function __construct( IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger, - private IFilesMetadataManager $filesMetadataManager, ) { parent::__construct($connection, $systemConfig, $logger); } @@ -140,8 +139,8 @@ public function whereParentInParameter(string $parameter): self { * * @return IMetadataQuery */ - public function selectMetadata(): IMetadataQuery { - $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + public function selectMetadata(IFilesMetadataManager $metadataManager): IMetadataQuery { + $metadataQuery = $metadataManager->getMetadataQuery($this, $this->alias, 'fileid'); $metadataQuery->retrieveMetadata(); return $metadataQuery; } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 849135d44f00e..3256ed0f1c4d7 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -55,6 +55,7 @@ public function __construct( private QueryOptimizer $queryOptimizer, private IGroupManager $groupManager, private IFilesMetadataManager $filesMetadataManager, + private CacheDatabase $cacheDatabase, ) { } @@ -106,15 +107,17 @@ protected function applySearchConstraints( * @return array */ public function findUsedTagsInCaches(ISearchQuery $searchQuery, array $caches): array { - $query = $this->getQueryBuilder(); - $query->selectTagUsage(); + $storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches); + $cachesByStorage = array_combine($storageIds, $caches); - $this->applySearchConstraints($query, $searchQuery, $caches); + return $this->cacheDatabase->queryStorages($storageIds, function(CacheQueryBuilder $query, $storages) use ($searchQuery, $cachesByStorage) { + $cachesForShard = array_map(fn (int $storage) => $cachesByStorage[$storage], $storages); + $query->selectTagUsage(); - $result = $query->execute(); - $tags = $result->fetchAll(); - $result->closeCursor(); - return $tags; + $this->applySearchConstraints($query, $searchQuery, $cachesForShard); + + return $query->executeQuery()->fetchAll(); + }); } protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user): void { @@ -171,35 +174,40 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array // while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id) // we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result. - $builder = $this->getQueryBuilder(); - - $query = $builder->selectFileCache('file', false); + $storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches); + $cachesByStorage = array_combine($storageIds, $caches); $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); - if (in_array('systemtag', $requestedFields)) { - $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); - } - if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { - $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); - } - if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { - $this->equipQueryForShares($query); - } + $rawEntries = $this->cacheDatabase->queryStorages($storageIds, function(CacheQueryBuilder $builder, $storages) use ($requestedFields, $searchQuery, $cachesByStorage) { + $cachesForShard = array_map(fn (int $storage) => $cachesByStorage[$storage], $storages); + $query = $builder->selectFileCache('file', false); + if (in_array('systemtag', $requestedFields)) { + $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); + } + if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { + $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); + } + if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { + $this->equipQueryForShares($query); + } + + $metadataQuery = $query->selectMetadata($this->filesMetadataManager); + + $this->applySearchConstraints($query, $searchQuery, $cachesForShard, $metadataQuery); + + $files = $query->executeQuery()->fetchAll(); - $metadataQuery = $query->selectMetadata(); + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); - $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); + return Cache::cacheEntryFromData($data, $this->mimetypeLoader); + }, $files); - $result = $query->execute(); - $files = $result->fetchAll(); + return $rawEntries; + }); - $rawEntries = array_map(function (array $data) use ($metadataQuery) { - $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); - return Cache::cacheEntryFromData($data, $this->mimetypeLoader); - }, $files); - $result->closeCursor(); // loop through all caches for each result to see if the result matches that storage // results are grouped by the same array keys as the caches argument to allow the caller to distinguish the source of the results diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php index 08c1b4f459c6e..e12f6af78121f 100644 --- a/lib/private/FilesMetadata/FilesMetadataManager.php +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -157,8 +157,8 @@ public function getMetadata(int $fileId, bool $generate = false): IFilesMetadata * @psalm-return array * @since 28.0.0 */ - public function getMetadataForFiles(array $fileIds): array { - return $this->metadataRequestService->getMetadataFromFileIds($fileIds); + public function getMetadataForFiles(array $fileIds, array $storageIds = []): array { + return $this->metadataRequestService->getMetadataFromFileIds($fileIds, $storageIds); } /** diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php index b6d1b277a0041..f70d939b8dd1b 100644 --- a/lib/private/FilesMetadata/Service/MetadataRequestService.php +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -25,6 +25,8 @@ namespace OC\FilesMetadata\Service; +use OC\Files\Cache\CacheDatabase; +use OC\Files\Cache\CacheQueryBuilder; use OC\FilesMetadata\Model\FilesMetadata; use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -41,7 +43,8 @@ class MetadataRequestService { public function __construct( private IDBConnection $dbConnection, - private LoggerInterface $logger + private LoggerInterface $logger, + private CacheDatabase $cacheDatabase, ) { } @@ -53,7 +56,8 @@ public function __construct( * @throws Exception */ public function store(IFilesMetadata $filesMetadata): void { - $qb = $this->dbConnection->getQueryBuilder(); + $file = $this->cacheDatabase->getByFileId($filesMetadata->getFileId()); + $qb = $this->cacheDatabase->queryForStorageId($file->getStorageId()); $qb->insert(self::TABLE_METADATA) ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) @@ -72,7 +76,12 @@ public function store(IFilesMetadata $filesMetadata): void { */ public function getMetadataFromFileId(int $fileId): IFilesMetadata { try { - $qb = $this->dbConnection->getQueryBuilder(); + $file = $this->cacheDatabase->getByFileId($fileId); + if (!$file) { + return new FilesMetadata($fileId); + } + $qb = $this->cacheDatabase->queryForStorageId($file->getStorageId()); + $qb->select('json', 'sync_token')->from(self::TABLE_METADATA); $qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); $result = $qb->executeQuery(); @@ -101,26 +110,43 @@ public function getMetadataFromFileId(int $fileId): IFilesMetadata { * @return array File ID is the array key, files without metadata are not returned in the array * @psalm-return array */ - public function getMetadataFromFileIds(array $fileIds): array { - $qb = $this->dbConnection->getQueryBuilder(); - $qb->select('file_id', 'json', 'sync_token')->from(self::TABLE_METADATA); - $qb->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); - - $list = []; - $result = $qb->executeQuery(); - while ($data = $result->fetch()) { - $fileId = (int) $data['file_id']; - $metadata = new FilesMetadata($fileId); - try { - $metadata->importFromDatabase($data); - } catch (FilesMetadataNotFoundException) { - continue; - } - $list[$fileId] = $metadata; + public function getMetadataFromFileIds(array $fileIds, array $storageIds = []): array { + if (!$storageIds) { + $files = $this->cacheDatabase->getByFileIds($fileIds); + $storageIds = array_map(fn (int $fileId) => $files[$fileId]->getStorageId(), $fileIds); } - $result->closeCursor(); + $storageByFile = array_combine($fileIds, $storageIds); + + $fileIdsByStorage = []; + foreach ($storageByFile as $fileId => $storage) { + $fileIdsByStorage[$storage][] = $fileId; + } + + return $this->cacheDatabase->queryStorages($storageIds, function(CacheQueryBuilder $qb, array $storages) use ($fileIdsByStorage) { + $fileIds = []; + foreach ($storages as $storage) { + $fileIds += $fileIdsByStorage[$storage]; + } + + $qb->select('file_id', 'json', 'sync_token')->from(self::TABLE_METADATA); + $qb->where( + $qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)) + ); - return $list; + $list = []; + $result = $qb->executeQuery(); + while ($data = $result->fetch()) { + $fileId = (int) $data['file_id']; + $metadata = new FilesMetadata($fileId); + try { + $metadata->importFromDatabase($data); + } catch (FilesMetadataNotFoundException) { + continue; + } + $list[$fileId] = $metadata; + } + return $list; + }); } /** @@ -132,10 +158,14 @@ public function getMetadataFromFileIds(array $fileIds): array { * @throws Exception */ public function dropMetadata(int $fileId): void { - $qb = $this->dbConnection->getQueryBuilder(); - $qb->delete(self::TABLE_METADATA) - ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); - $qb->executeStatement(); + $file = $this->cacheDatabase->getByFileId($fileId); + if ($file) { + $qb = $this->cacheDatabase->queryForStorageId($file->getStorageId()); + + $qb->delete(self::TABLE_METADATA) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + } } /** @@ -147,7 +177,8 @@ public function dropMetadata(int $fileId): void { * @throws Exception */ public function updateMetadata(IFilesMetadata $filesMetadata): int { - $qb = $this->dbConnection->getQueryBuilder(); + $file = $this->cacheDatabase->getByFileId($filesMetadata->getFileId()); + $qb = $this->cacheDatabase->queryForStorageId($file->getStorageId()); $expr = $qb->expr(); $qb->update(self::TABLE_METADATA) diff --git a/lib/private/Share20/DefaultShareProvider.php b/lib/private/Share20/DefaultShareProvider.php index 318c898621967..31060805bde8c 100644 --- a/lib/private/Share20/DefaultShareProvider.php +++ b/lib/private/Share20/DefaultShareProvider.php @@ -41,6 +41,7 @@ use OCP\AppFramework\Utility\ITimeFactory; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Defaults; +use OCP\Files\Cache\IFileAccess; use OCP\Files\Folder; use OCP\Files\IRootFolder; use OCP\Files\Node; @@ -67,52 +68,21 @@ class DefaultShareProvider implements IShareProvider { // Special share type for user modified group shares public const SHARE_TYPE_USERGROUP = 2; - /** @var IDBConnection */ - private $dbConn; - - /** @var IUserManager */ - private $userManager; - - /** @var IGroupManager */ - private $groupManager; - - /** @var IRootFolder */ - private $rootFolder; - - /** @var IMailer */ - private $mailer; - - /** @var Defaults */ - private $defaults; - - /** @var IFactory */ - private $l10nFactory; - - /** @var IURLGenerator */ - private $urlGenerator; - - private ITimeFactory $timeFactory; + private IDBConnection $dbConn; public function __construct( IDBConnection $connection, - IUserManager $userManager, - IGroupManager $groupManager, - IRootFolder $rootFolder, - IMailer $mailer, - Defaults $defaults, - IFactory $l10nFactory, - IURLGenerator $urlGenerator, - ITimeFactory $timeFactory, + private IUserManager $userManager, + private IGroupManager $groupManager, + private IRootFolder $rootFolder, + private IMailer $mailer, + private Defaults $defaults, + private IFactory $l10nFactory, + private IURLGenerator $urlGenerator, + private ITimeFactory $timeFactory, + private IFileAccess $fileAccess, ) { $this->dbConn = $connection; - $this->userManager = $userManager; - $this->groupManager = $groupManager; - $this->rootFolder = $rootFolder; - $this->mailer = $mailer; - $this->defaults = $defaults; - $this->l10nFactory = $l10nFactory; - $this->urlGenerator = $urlGenerator; - $this->timeFactory = $timeFactory; } /** @@ -891,15 +861,8 @@ public function getSharedWith($userId, $shareType, $node, $limit, $offset) { if ($shareType === IShare::TYPE_USER) { //Get shares directly with this user $qb = $this->dbConn->getQueryBuilder(); - $qb->select('s.*', - 'f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', - 'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime', - 'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum' - ) - ->selectAlias('st.id', 'storage_string_id') - ->from('share', 's') - ->leftJoin('s', 'filecache', 'f', $qb->expr()->eq('s.file_source', 'f.fileid')) - ->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id')); + $qb->select('s.*') + ->from('share', 's'); // Order by id $qb->orderBy('s.id'); @@ -925,14 +888,7 @@ public function getSharedWith($userId, $shareType, $node, $limit, $offset) { $cursor = $qb->execute(); while ($data = $cursor->fetch()) { - if ($data['fileid'] && $data['path'] === null) { - $data['path'] = (string) $data['path']; - $data['name'] = (string) $data['name']; - $data['checksum'] = (string) $data['checksum']; - } - if ($this->isAccessibleResult($data)) { - $shares[] = $this->createShare($data); - } + $shares[] = $this->createShare($data); } $cursor->closeCursor(); } elseif ($shareType === IShare::TYPE_GROUP) { @@ -952,15 +908,8 @@ public function getSharedWith($userId, $shareType, $node, $limit, $offset) { } $qb = $this->dbConn->getQueryBuilder(); - $qb->select('s.*', - 'f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', - 'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime', - 'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum' - ) - ->selectAlias('st.id', 'storage_string_id') + $qb->select('s.*') ->from('share', 's') - ->leftJoin('s', 'filecache', 'f', $qb->expr()->eq('s.file_source', 'f.fileid')) - ->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id')) ->orderBy('s.id') ->setFirstResult(0); @@ -1010,7 +959,31 @@ public function getSharedWith($userId, $shareType, $node, $limit, $offset) { } - return $shares; + return $this->setNodes($shares); + } + + /** + * @param IShare[] $shares + * @return IShare[] + */ + private function setNodes(array $shares): array { + $fileIds = array_map(function (IShare $share): int { + return $share->getNodeId(); + }, $shares); + $files = $this->fileAccess->getByFileIds($fileIds); + + $sharesWithFiles = []; + foreach ($shares as $share) { + if (isset($files[$share->getNodeId()])) { + $cacheItem = $files[$share->getNodeId()]; + if ($this->isAccessibleResult($cacheItem->getData())) { + $share->setNodeCacheEntry($cacheItem); + $sharesWithFiles[] = $share; + } + } + } + + return $sharesWithFiles; } /** diff --git a/lib/private/Share20/ProviderFactory.php b/lib/private/Share20/ProviderFactory.php index cac25226f1ae9..c193f9fda0393 100644 --- a/lib/private/Share20/ProviderFactory.php +++ b/lib/private/Share20/ProviderFactory.php @@ -99,17 +99,7 @@ public function registerProvider(string $shareProviderClass): void { */ protected function defaultShareProvider() { if ($this->defaultProvider === null) { - $this->defaultProvider = new DefaultShareProvider( - $this->serverContainer->getDatabaseConnection(), - $this->serverContainer->getUserManager(), - $this->serverContainer->getGroupManager(), - $this->serverContainer->get(IRootFolder::class), - $this->serverContainer->getMailer(), - $this->serverContainer->query(Defaults::class), - $this->serverContainer->getL10NFactory(), - $this->serverContainer->getURLGenerator(), - $this->serverContainer->query(ITimeFactory::class), - ); + $this->defaultProvider = $this->serverContainer->get(DefaultShareProvider::class); } return $this->defaultProvider; diff --git a/lib/public/FilesMetadata/IFilesMetadataManager.php b/lib/public/FilesMetadata/IFilesMetadataManager.php index 6dee48c264be2..7f73631debf3a 100644 --- a/lib/public/FilesMetadata/IFilesMetadataManager.php +++ b/lib/public/FilesMetadata/IFilesMetadataManager.php @@ -86,12 +86,13 @@ public function getMetadata(int $fileId, bool $generate = false): IFilesMetadata * returns metadata of multiple file ids * * @param int[] $fileIds file ids + * @param int[] $storageIds storage ids for the files * * @return array File ID is the array key, files without metadata are not returned in the array * @psalm-return array * @since 28.0.0 */ - public function getMetadataForFiles(array $fileIds): array; + public function getMetadataForFiles(array $fileIds, array $storageIds = []): array; /** * save metadata to database and refresh indexes. From f481a4af78f15a336e63f9b9ecc378918ebee11b Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 7 Mar 2024 11:59:42 +0100 Subject: [PATCH 5/7] refactor: don't join on filecache in usermountcache Signed-off-by: Robin Appelman --- lib/private/Files/Config/UserMountCache.php | 109 +++++++----------- tests/lib/Files/Config/UserMountCacheTest.php | 72 +++++++----- 2 files changed, 87 insertions(+), 94 deletions(-) diff --git a/lib/private/Files/Config/UserMountCache.php b/lib/private/Files/Config/UserMountCache.php index b5b07d8f29e51..172d85d43aaee 100644 --- a/lib/private/Files/Config/UserMountCache.php +++ b/lib/private/Files/Config/UserMountCache.php @@ -26,8 +26,10 @@ * along with this program. If not, see * */ + namespace OC\Files\Config; +use OC\Files\Cache\FileAccess; use OC\User\LazyUser; use OCP\Cache\CappedMemoryCache; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -45,37 +47,31 @@ * Cache mounts points per user in the cache so we can easily look them up */ class UserMountCache implements IUserMountCache { - private IDBConnection $connection; - private IUserManager $userManager; - /** * Cached mount info. + * * @var CappedMemoryCache **/ private CappedMemoryCache $mountsForUsers; /** * fileid => internal path mapping for cached mount info. + * * @var CappedMemoryCache **/ private CappedMemoryCache $internalPathCache; - private LoggerInterface $logger; /** @var CappedMemoryCache */ private CappedMemoryCache $cacheInfoCache; - private IEventLogger $eventLogger; /** * UserMountCache constructor. */ public function __construct( - IDBConnection $connection, - IUserManager $userManager, - LoggerInterface $logger, - IEventLogger $eventLogger + private IDBConnection $connection, + private IUserManager $userManager, + private LoggerInterface $logger, + private IEventLogger $eventLogger, + private FileAccess $cacheAccess, ) { - $this->connection = $connection; - $this->userManager = $userManager; - $this->logger = $logger; - $this->eventLogger = $eventLogger; $this->cacheInfoCache = new CappedMemoryCache(); $this->internalPathCache = new CappedMemoryCache(); $this->mountsForUsers = new CappedMemoryCache(); @@ -282,11 +278,8 @@ public function getInternalPathForMountInfo(CachedMountInfo $info): string { if ($cached !== null) { return $cached; } - $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('path') - ->from('filecache') - ->where($builder->expr()->eq('fileid', $builder->createPositionalParameter($info->getRootId()))); - return $query->executeQuery()->fetchOne() ?: ''; + $entry = $this->cacheAccess->getByFileIdInStorage($info->getRootId(), $info->getStorageId()); + return $entry ? $entry->getPath() : ''; } /** @@ -294,11 +287,10 @@ public function getInternalPathForMountInfo(CachedMountInfo $info): string { * @param string|null $user limit the results to a single user * @return CachedMountInfo[] */ - public function getMountsForStorageId($numericStorageId, $user = null) { + public function getMountsForStorageId($numericStorageId, $user = null, bool $preloadPaths = false) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class') + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'mount_provider_class') ->from('mounts', 'm') - ->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid')) ->where($builder->expr()->eq('storage_id', $builder->createPositionalParameter($numericStorageId, IQueryBuilder::PARAM_INT))); if ($user) { @@ -309,7 +301,21 @@ public function getMountsForStorageId($numericStorageId, $user = null) { $rows = $result->fetchAll(); $result->closeCursor(); - return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows)); + if ($preloadPaths) { + $fileIds = array_map(fn (array $row) => $row['root_id'], $rows); + $files = $this->cacheAccess->getByFileIds($fileIds); + + foreach ($rows as &$row) { + $mountFileId = $row['root_id']; + if (isset($files[$mountFileId])) { + $row['path'] = $files[$mountFileId]->getPath(); + } + } + } + + return array_filter(array_map(function (array $row) use ($preloadPaths) { + return $this->dbRowToMountInfo($row, $preloadPaths ? null : [$this, 'getInternalPathForMountInfo']); + }, $rows)); } /** @@ -318,45 +324,17 @@ public function getMountsForStorageId($numericStorageId, $user = null) { */ public function getMountsForRootId($rootFileId) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class') + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'mount_provider_class') ->from('mounts', 'm') - ->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid')) ->where($builder->expr()->eq('root_id', $builder->createPositionalParameter($rootFileId, IQueryBuilder::PARAM_INT))); $result = $query->execute(); $rows = $result->fetchAll(); $result->closeCursor(); - return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows)); - } - - /** - * @param $fileId - * @return array{int, string, int} - * @throws \OCP\Files\NotFoundException - */ - private function getCacheInfoFromFileId($fileId): array { - if (!isset($this->cacheInfoCache[$fileId])) { - $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage', 'path', 'mimetype') - ->from('filecache') - ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); - - $result = $query->execute(); - $row = $result->fetch(); - $result->closeCursor(); - - if (is_array($row)) { - $this->cacheInfoCache[$fileId] = [ - (int)$row['storage'], - (string)$row['path'], - (int)$row['mimetype'] - ]; - } else { - throw new NotFoundException('File with id "' . $fileId . '" not found'); - } - } - return $this->cacheInfoCache[$fileId]; + return array_filter(array_map(function (array $row) { + return $this->dbRowToMountInfo($row, [$this, 'getInternalPathForMountInfo']); + }, $rows)); } /** @@ -366,12 +344,13 @@ private function getCacheInfoFromFileId($fileId): array { * @since 9.0.0 */ public function getMountsForFileId($fileId, $user = null) { - try { - [$storageId, $internalPath] = $this->getCacheInfoFromFileId($fileId); - } catch (NotFoundException $e) { + $cacheEntry = $this->cacheAccess->getByFileId($fileId); + if (!$cacheEntry) { return []; } - $mountsForStorage = $this->getMountsForStorageId($storageId, $user); + $internalPath = $cacheEntry->getPath(); + + $mountsForStorage = $this->getMountsForStorageId($cacheEntry->getStorageId(), $user, true); // filter mounts that are from the same storage but not a parent of the file we care about $filteredMounts = array_filter($mountsForStorage, function (ICachedMountInfo $mount) use ($internalPath, $fileId) { @@ -449,13 +428,8 @@ public function getUsedSpaceForUsers(array $users) { return $user->getUID(); }, $users); - $query = $builder->select('m.user_id', 'f.size') + $query = $builder->select('m.user_id', 'storage_id') ->from('mounts', 'm') - ->innerJoin('m', 'filecache', 'f', - $builder->expr()->andX( - $builder->expr()->eq('m.storage_id', 'f.storage'), - $builder->expr()->eq('f.path_hash', $builder->createNamedParameter(md5('files'))) - )) ->where($builder->expr()->eq('m.mount_point', $mountPoint)) ->andWhere($builder->expr()->in('m.user_id', $builder->createNamedParameter($userIds, IQueryBuilder::PARAM_STR_ARRAY))); @@ -463,12 +437,17 @@ public function getUsedSpaceForUsers(array $users) { $results = []; while ($row = $result->fetch()) { - $results[$row['user_id']] = $row['size']; + $results[$row['user_id']] = $this->getSizeForHomeStorage($row['storage_id']); } $result->closeCursor(); return $results; } + private function getSizeForHomeStorage(int $storage): int { + $entry = $this->cacheAccess->getByPathInStorage('files', $storage); + return $entry->getSize(); + } + public function clear(): void { $this->cacheInfoCache = new CappedMemoryCache(); $this->mountsForUsers = new CappedMemoryCache(); diff --git a/tests/lib/Files/Config/UserMountCacheTest.php b/tests/lib/Files/Config/UserMountCacheTest.php index 13690096d3a5e..9202d5895377f 100644 --- a/tests/lib/Files/Config/UserMountCacheTest.php +++ b/tests/lib/Files/Config/UserMountCacheTest.php @@ -7,11 +7,14 @@ namespace Test\Files\Config; +use OC\DB\Exceptions\DbalException; use OC\DB\QueryBuilder\Literal; +use OC\Files\Cache\FileAccess; use OC\Files\Mount\MountPoint; use OC\Files\Storage\Storage; use OC\User\Manager; use OCP\Cache\CappedMemoryCache; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Diagnostics\IEventLogger; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Config\ICachedMountInfo; @@ -19,6 +22,7 @@ use OCP\IConfig; use OCP\IDBConnection; use OCP\IUserManager; +use OCP\Server; use Psr\Log\LoggerInterface; use Test\TestCase; use Test\Util\User\Dummy; @@ -27,10 +31,8 @@ * @group DB */ class UserMountCacheTest extends TestCase { - /** - * @var IDBConnection - */ - private $connection; + private IDBConnection $connection; + private FileAccess $cacheAccess; /** * @var IUserManager @@ -48,7 +50,8 @@ protected function setUp(): void { parent::setUp(); $this->fileIds = []; - $this->connection = \OC::$server->getDatabaseConnection(); + $this->connection = Server::get(IDBConnection::class); + $this->cacheAccess = Server::get(FileAccess::class); $config = $this->getMockBuilder(IConfig::class) ->disableOriginalConstructor() ->getMock(); @@ -66,7 +69,13 @@ protected function setUp(): void { $userBackend->createUser('u2', ''); $userBackend->createUser('u3', ''); $this->userManager->registerBackend($userBackend); - $this->cache = new \OC\Files\Config\UserMountCache($this->connection, $this->userManager, $this->createMock(LoggerInterface::class), $this->createMock(IEventLogger::class)); + $this->cache = new \OC\Files\Config\UserMountCache( + $this->connection, + $this->userManager, + $this->createMock(LoggerInterface::class), + $this->createMock(IEventLogger::class), + $this->cacheAccess, + ); } protected function tearDown(): void { @@ -335,31 +344,36 @@ private function sortMounts(&$mounts) { private function createCacheEntry($internalPath, $storageId, $size = 0) { $internalPath = trim($internalPath, '/'); - $inserted = $this->connection->insertIfNotExist('*PREFIX*filecache', [ - 'storage' => $storageId, - 'path' => $internalPath, - 'path_hash' => md5($internalPath), - 'parent' => -1, - 'name' => basename($internalPath), - 'mimetype' => 0, - 'mimepart' => 0, - 'size' => $size, - 'storage_mtime' => 0, - 'encrypted' => 0, - 'unencrypted_size' => 0, - 'etag' => '', - 'permissions' => 31 - ], ['storage', 'path_hash']); - if ($inserted) { - $id = (int)$this->connection->lastInsertId('*PREFIX*filecache'); + $query = $this->connection->getQueryBuilder(); + $query->insert('filecache') + ->values([ + 'storage' => $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT), + 'path' => $query->createNamedParameter($internalPath), + 'path_hash' => $query->createNamedParameter(md5($internalPath)), + 'parent' => $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT), + 'name' => $query->createNamedParameter(basename($internalPath)), + 'mimetype' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT), + 'mimepart' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT), + 'size' => $query->createNamedParameter($size, IQueryBuilder::PARAM_INT), + 'storage_mtime' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT), + 'encrypted' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT), + 'unencrypted_size' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT), + 'etag' => $query->createNamedParameter(''), + 'permissions' => $query->createNamedParameter(31, IQueryBuilder::PARAM_INT), + ]); + try { + $query->executeStatement(); + $id = (int)$query->getLastInsertId(); $this->fileIds[] = $id; - } else { - $sql = 'SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` =?'; - $query = $this->connection->prepare($sql); - $query->execute([$storageId, md5($internalPath)]); - return (int)$query->fetchOne(); + return $id; + } catch (DbalException $e) { + $query = $this->connection->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $query->createNamedParameter(md5($internalPath)))); + return (int)$query->executeQuery()->fetchOne(); } - return $id; } public function testGetMountsForFileIdRootId() { From e7c884043aca8802bd40aadcd4ecdf9be706085d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 1 Mar 2024 20:07:26 +0100 Subject: [PATCH 6/7] transform favorite, tag and systemtag search queries to fileid lists not as performant but compatible with sharding Signed-off-by: Robin Appelman --- lib/private/Files/Cache/QuerySearchHelper.php | 66 +++++++++-------- lib/private/Files/Cache/SearchBuilder.php | 16 ---- .../Search/QueryOptimizer/DavTagToFileIds.php | 70 ++++++++++++++++++ .../Search/QueryOptimizer/FavoriteToTag.php | 44 +++++++++++ .../QueryOptimizer/SystemTagToFileIds.php | 73 +++++++++++++++++++ lib/private/Files/Search/SearchQuery.php | 4 + lib/public/Files/Search/ISearchQuery.php | 7 ++ 7 files changed, 234 insertions(+), 46 deletions(-) create mode 100644 lib/private/Files/Search/QueryOptimizer/DavTagToFileIds.php create mode 100644 lib/private/Files/Search/QueryOptimizer/FavoriteToTag.php create mode 100644 lib/private/Files/Search/QueryOptimizer/SystemTagToFileIds.php diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 3256ed0f1c4d7..61165d29715b0 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -24,11 +24,17 @@ * along with this program. If not, see . * */ + namespace OC\Files\Cache; use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Search\QueryOptimizer\DavTagToFileIds; +use OC\Files\Search\QueryOptimizer\FavoriteToTag; use OC\Files\Search\QueryOptimizer\QueryOptimizer; +use OC\Files\Search\QueryOptimizer\SystemTagToFileIds; use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchQuery; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\ICache; @@ -47,13 +53,13 @@ class QuerySearchHelper { public function __construct( - private IMimeTypeLoader $mimetypeLoader, - private IDBConnection $connection, - private SystemConfig $systemConfig, - private LoggerInterface $logger, - private SearchBuilder $searchBuilder, - private QueryOptimizer $queryOptimizer, - private IGroupManager $groupManager, + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, private IFilesMetadataManager $filesMetadataManager, private CacheDatabase $cacheDatabase, ) { @@ -76,9 +82,9 @@ protected function getQueryBuilder() { */ protected function applySearchConstraints( CacheQueryBuilder $query, - ISearchQuery $searchQuery, - array $caches, - ?IMetadataQuery $metadataQuery = null + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null ): void { $storageFilters = array_values(array_map(function (ICache $cache) { return $cache->getQueryFilterForStorage(); @@ -132,18 +138,6 @@ protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user $query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on); } - protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void { - $query - ->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid')) - ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX( - $query->expr()->eq('tagmap.type', 'tag.type'), - $query->expr()->eq('tagmap.categoryid', 'tag.id'), - $query->expr()->eq('tag.type', $query->createNamedParameter('files')), - $query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID())) - )); - } - - protected function equipQueryForShares(CacheQueryBuilder $query): void { $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source')); } @@ -174,20 +168,32 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array // while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id) // we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result. - $storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches); - $cachesByStorage = array_combine($storageIds, $caches); + $operation = $searchQuery->getSearchOperation(); $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + if (in_array('favorite', $requestedFields)) { + $favoriteHandler = new FavoriteToTag(); + $favoriteHandler->processOperator($operation); + } + + if (in_array('systemtag', $requestedFields)) { + $systemTagHandler = new SystemTagToFileIds($this->connection, $this->requireUser($searchQuery), $this->groupManager); + $systemTagHandler->processOperator($operation); + } + if (in_array('tagname', $requestedFields)) { + $davTagHandler = new DavTagToFileIds($this->connection, $this->requireUser($searchQuery)); + $davTagHandler->processOperator($operation); + } + + $searchQuery->setSearchOperation($operation); + + $storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches); + $cachesByStorage = array_combine($storageIds, $caches); + $rawEntries = $this->cacheDatabase->queryStorages($storageIds, function(CacheQueryBuilder $builder, $storages) use ($requestedFields, $searchQuery, $cachesByStorage) { $cachesForShard = array_map(fn (int $storage) => $cachesByStorage[$storage], $storages); $query = $builder->selectFileCache('file', false); - if (in_array('systemtag', $requestedFields)) { - $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); - } - if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { - $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); - } if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { $this->equipQueryForShares($query); } diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php index e8063b77aa597..633e23f764707 100644 --- a/lib/private/Files/Cache/SearchBuilder.php +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -75,9 +75,6 @@ class SearchBuilder { 'name' => 'string', 'path' => 'string', 'size' => 'integer', - 'tagname' => 'string', - 'systemtag' => 'string', - 'favorite' => 'boolean', 'fileid' => 'integer', 'storage' => 'integer', 'share_with' => 'string', @@ -99,8 +96,6 @@ class SearchBuilder { 'boolean' => IQueryBuilder::PARAM_INT_ARRAY, ]; - public const TAG_FAVORITE = '_$!!$_'; - /** @var IMimeTypeLoader */ private $mimetypeLoader; @@ -243,16 +238,8 @@ private function getOperatorFieldAndValueInner(string $field, mixed $value, stri $type = ISearchComparison::COMPARE_EQUAL; } } - } elseif ($field === 'favorite') { - $field = 'tag.category'; - $value = self::TAG_FAVORITE; - $paramType = 'string'; } elseif ($field === 'name') { $field = 'file.name'; - } elseif ($field === 'tagname') { - $field = 'tag.category'; - } elseif ($field === 'systemtag') { - $field = 'systemtag.name'; } elseif ($field === 'fileid') { $field = 'file.fileid'; } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) { @@ -271,9 +258,6 @@ private function validateComparison(ISearchComparison $operator) { 'name' => ['eq', 'like', 'clike', 'in'], 'path' => ['eq', 'like', 'clike', 'in'], 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], - 'tagname' => ['eq', 'like'], - 'systemtag' => ['eq', 'like'], - 'favorite' => ['eq'], 'fileid' => ['eq', 'in'], 'storage' => ['eq', 'in'], 'share_with' => ['eq'], diff --git a/lib/private/Files/Search/QueryOptimizer/DavTagToFileIds.php b/lib/private/Files/Search/QueryOptimizer/DavTagToFileIds.php new file mode 100644 index 0000000000000..439bdc0a7f2b5 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/DavTagToFileIds.php @@ -0,0 +1,70 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; + +class DavTagToFileIds extends ReplacingOptimizerStep { + public function __construct( + private IDBConnection $connection, + private IUser $user, + ) { + } + + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof ISearchComparison && $operator->getField() === 'tagname') { + $operator = new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $this->getFileIdsForDavTag($operator)); + return true; + } else { + return parent::processOperator($operator); + } + } + + + private function getFileIdsForDavTag(ISearchComparison $comparison): array { + $query = $this->connection->getQueryBuilder(); + + $query->select('tagmap.objid') + ->from('vcategory_to_object', 'tagmap') + ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX( + $query->expr()->eq('tagmap.type', 'tag.type'), + $query->expr()->eq('tagmap.categoryid', 'tag.id') + )) + ->where($query->expr()->eq('tag.type', $query->createNamedParameter('files'))) + ->andWhere($query->expr()->eq('tag.uid', $query->createNamedParameter($this->user->getUID()))); + if ($comparison->getType() === ISearchComparison::COMPARE_EQUAL) { + $query->andWhere($query->expr()->eq('tag.category', $query->createNamedParameter($comparison->getValue()))); + } elseif ($comparison->getType() === ISearchComparison::COMPARE_LIKE) { + $query->andWhere($query->expr()->like('tag.category', $query->createNamedParameter($comparison->getValue()))); + } else { + throw new \InvalidArgumentException('Unsupported comparison for field ' . $comparison->getField() . ': ' . $comparison->getType()); + } + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/FavoriteToTag.php b/lib/private/Files/Search/QueryOptimizer/FavoriteToTag.php new file mode 100644 index 0000000000000..50157829c7690 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/FavoriteToTag.php @@ -0,0 +1,44 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; + +class FavoriteToTag extends ReplacingOptimizerStep { + public const TAG_FAVORITE = '_$!!$_'; + + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof ISearchComparison && $operator->getField() === 'favorite' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { + $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'tagname', self::TAG_FAVORITE); + return true; + } else { + return parent::processOperator($operator); + } + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/SystemTagToFileIds.php b/lib/private/Files/Search/QueryOptimizer/SystemTagToFileIds.php new file mode 100644 index 0000000000000..60c9372f2485b --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/SystemTagToFileIds.php @@ -0,0 +1,73 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; + +class SystemTagToFileIds extends ReplacingOptimizerStep { + public function __construct( + private IDBConnection $connection, + private IUser $user, + private IGroupManager $groupManager, + ) { + } + + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof ISearchComparison && $operator->getField() === 'systemtag') { + $operator = new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $this->getFileIdsForSystemTag($operator)); + return true; + } else { + return parent::processOperator($operator); + } + } + + + private function getFileIdsForSystemTag(ISearchComparison $comparison): array { + $query = $this->connection->getQueryBuilder(); + + $on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid')); + if (!$this->groupManager->isAdmin($this->user->getUID())) { + $on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true))); + } + + $query->select('systemtagmap.objectid') + ->from('systemtag_object_mapping', 'systemtagmap') + ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on) + ->where($query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files'))) + ->andWhere($query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files'))); + if ($comparison->getType() === ISearchComparison::COMPARE_EQUAL) { + $query->andWhere($query->expr()->eq('systemtag.name', $query->createNamedParameter($comparison->getValue()))); + } elseif ($comparison->getType() === ISearchComparison::COMPARE_LIKE) { + $query->andWhere($query->expr()->like('systemtag.name', $query->createNamedParameter($comparison->getValue()))); + } else { + throw new \InvalidArgumentException('Unsupported comparison for field ' . $comparison->getField() . ': ' . $comparison->getType()); + } + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } +} diff --git a/lib/private/Files/Search/SearchQuery.php b/lib/private/Files/Search/SearchQuery.php index 7c76bdff97833..b33ccea953f72 100644 --- a/lib/private/Files/Search/SearchQuery.php +++ b/lib/private/Files/Search/SearchQuery.php @@ -73,6 +73,10 @@ public function getSearchOperation() { return $this->searchOperation; } + public function setSearchOperation(ISearchOperator $operator): void { + $this->searchOperation = $operator; + } + /** * @return int */ diff --git a/lib/public/Files/Search/ISearchQuery.php b/lib/public/Files/Search/ISearchQuery.php index 91c6f6425666c..58a942bb1ae61 100644 --- a/lib/public/Files/Search/ISearchQuery.php +++ b/lib/public/Files/Search/ISearchQuery.php @@ -34,6 +34,13 @@ interface ISearchQuery { */ public function getSearchOperation(); + /** + * @param ISearchOperator $operator + * @return void + * @since 29.0.0 + */ + public function setSearchOperation(ISearchOperator $operator): void; + /** * Get the maximum number of results to return * From 4c6aabf03f8b5e02b2a2fe4f2f6b74befcb86a5d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 22 May 2024 17:11:29 +0200 Subject: [PATCH 7/7] register DI alias for IFileAccess Signed-off-by: Robin Appelman --- lib/private/Server.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/private/Server.php b/lib/private/Server.php index 8f2fb2b7383ea..7bd07ff085d36 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -86,6 +86,7 @@ use OC\Federation\CloudFederationFactory; use OC\Federation\CloudFederationProviderManager; use OC\Federation\CloudIdManager; +use OC\Files\Cache\FileAccess; use OC\Files\Config\MountProviderCollection; use OC\Files\Config\UserMountCache; use OC\Files\Config\UserMountCacheListener; @@ -195,6 +196,7 @@ use OCP\Federation\ICloudFederationFactory; use OCP\Federation\ICloudFederationProviderManager; use OCP\Federation\ICloudIdManager; +use OCP\Files\Cache\IFileAccess; use OCP\Files\Config\IMountProviderCollection; use OCP\Files\Config\IUserMountCache; use OCP\Files\IMimeTypeDetector; @@ -450,6 +452,7 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerService(ISystemTagObjectMapper::class, function (ContainerInterface $c) { return $c->get('SystemTagManagerFactory')->getObjectMapper(); }); + $this->registerAlias(IFileAccess::class, FileAccess::class); $this->registerService('RootFolder', function (ContainerInterface $c) { $manager = \OC\Files\Filesystem::getMountManager(); $view = new View();