From 367ba8bef44f352b23462863f527c3c583876e64 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 14 Feb 2024 16:05:34 +0100 Subject: [PATCH] add class for (future) shared access to the filecache Signed-off-by: Robin Appelman --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Files/Cache/Cache.php | 61 ++---- lib/private/Files/Cache/CacheDependencies.php | 5 + lib/private/Files/Cache/CacheQueryBuilder.php | 12 +- lib/private/Files/Cache/Database.php | 191 ++++++++++++++++++ lib/private/Files/Cache/Storage.php | 2 +- .../Files/Cache/Wrapper/CacheWrapper.php | 8 +- tests/lib/Files/Cache/CacheTest.php | 20 -- 9 files changed, 227 insertions(+), 74 deletions(-) create mode 100644 lib/private/Files/Cache/Database.php diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index c027075d1605a..01f5d46781606 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1343,6 +1343,7 @@ 'OC\\Files\\Cache\\CacheDependencies' => $baseDir . '/lib/private/Files/Cache/CacheDependencies.php', 'OC\\Files\\Cache\\CacheEntry' => $baseDir . '/lib/private/Files/Cache/CacheEntry.php', 'OC\\Files\\Cache\\CacheQueryBuilder' => $baseDir . '/lib/private/Files/Cache/CacheQueryBuilder.php', + 'OC\\Files\\Cache\\Database' => $baseDir . '/lib/private/Files/Cache/Database.php', 'OC\\Files\\Cache\\FailedCache' => $baseDir . '/lib/private/Files/Cache/FailedCache.php', 'OC\\Files\\Cache\\HomeCache' => $baseDir . '/lib/private/Files/Cache/HomeCache.php', 'OC\\Files\\Cache\\HomePropagator' => $baseDir . '/lib/private/Files/Cache/HomePropagator.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 238b667b5f4f3..698ffddf6559d 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1376,6 +1376,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\Cache\\CacheDependencies' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheDependencies.php', 'OC\\Files\\Cache\\CacheEntry' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheEntry.php', 'OC\\Files\\Cache\\CacheQueryBuilder' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheQueryBuilder.php', + 'OC\\Files\\Cache\\Database' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Database.php', 'OC\\Files\\Cache\\FailedCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FailedCache.php', 'OC\\Files\\Cache\\HomeCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/HomeCache.php', 'OC\\Files\\Cache\\HomePropagator' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/HomePropagator.php', diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index a512bf76ace33..e09b98de88a7a 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -61,7 +61,7 @@ use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; use OCP\FilesMetadata\IFilesMetadataManager; -use OCP\IDBConnection; +use OCP\Server; use OCP\Util; use Psr\Log\LoggerInterface; @@ -87,7 +87,7 @@ class Cache implements ICache { protected string $storageId; protected Storage $storageCache; protected IMimeTypeLoader$mimetypeLoader; - protected IDBConnection $connection; + protected Database $cacheDb; protected SystemConfig $systemConfig; protected LoggerInterface $logger; protected QuerySearchHelper $querySearchHelper; @@ -105,11 +105,11 @@ public function __construct( $this->storageId = md5($this->storageId); } if (!$dependencies) { - $dependencies = \OC::$server->get(CacheDependencies::class); + $dependencies = Server::get(CacheDependencies::class); } $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); - $this->connection = $dependencies->getConnection(); + $this->cacheDb = $dependencies->getCacheDb(); $this->systemConfig = $dependencies->getSystemConfig(); $this->logger = $dependencies->getLogger(); $this->querySearchHelper = $dependencies->getQuerySearchHelper(); @@ -118,12 +118,7 @@ public function __construct( } protected function getQueryBuilder() { - return new CacheQueryBuilder( - $this->connection, - $this->systemConfig, - $this->logger, - $this->metadataManager, - ); + return $this->cacheDb->queryForStorageId($this->getNumericStorageId()); } public function getStorageCache(): Storage { @@ -304,7 +299,7 @@ public function insert($file, array $data) { $values['storage'] = $storageId; try { - $builder = $this->connection->getQueryBuilder(); + $builder = $this->getQueryBuilder(); $builder->insert('filecache'); foreach ($values as $column => $value) { @@ -332,9 +327,9 @@ public function insert($file, array $data) { } } catch (UniqueConstraintViolationException $e) { // entry exists already - if ($this->connection->inTransaction()) { - $this->connection->commit(); - $this->connection->beginTransaction(); + if ($this->cacheDb->inTransaction($this->getNumericStorageId())) { + $this->cacheDb->commit($this->getNumericStorageId()); + $this->cacheDb->beginTransaction($this->getNumericStorageId()); } } @@ -687,11 +682,11 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { throw new \Exception('Invalid target storage id: ' . $targetStorageId); } - $this->connection->beginTransaction(); + $this->cacheDb->beginTransaction($this->getNumericStorageId()); if ($sourceData['mimetype'] === 'httpd/unix-directory') { //update all child entries $sourceLength = mb_strlen($sourcePath); - $query = $this->connection->getQueryBuilder(); + $query = $this->getQueryBuilder(); $fun = $query->func(); $newPathFunction = $fun->concat( @@ -703,7 +698,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { ->set('path_hash', $fun->md5($newPathFunction)) ->set('path', $newPathFunction) ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%'))); + ->andWhere($query->expr()->like('path', $query->createNamedParameter($query->escapeLikeParameter($sourcePath) . '/%'))); // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { @@ -713,7 +708,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { try { $query->execute(); } catch (\OC\DatabaseException $e) { - $this->connection->rollBack(); + $this->cacheDb->rollBack($this->getNumericStorageId()); throw $e; } } @@ -734,7 +729,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { $query->execute(); - $this->connection->commit(); + $this->cacheDb->commit($this->getNumericStorageId()); if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) { $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId())); @@ -760,7 +755,7 @@ public function clear() { ->whereStorageId($this->getNumericStorageId()); $query->execute(); - $query = $this->connection->getQueryBuilder(); + $query = $this->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); $query->execute(); @@ -834,8 +829,8 @@ public function searchByMime($mimetype) { return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** @@ -1072,27 +1067,7 @@ public function getPathById($id) { * @deprecated use getPathById() instead */ public static function getById($id) { - $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); - $query->select('path', 'storage') - ->from('filecache') - ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); - - $result = $query->execute(); - $row = $result->fetch(); - $result->closeCursor(); - - if ($row) { - $numericId = $row['storage']; - $path = $row['path']; - } else { - return null; - } - - if ($id = Storage::getStorageId($numericId)) { - return [$id, $path]; - } else { - return null; - } + throw new \Exception("Cache::getById has been removed"); } /** diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php index 7c51f3ff88421..83967eab72409 100644 --- a/lib/private/Files/Cache/CacheDependencies.php +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -20,6 +20,7 @@ public function __construct( private LoggerInterface $logger, private IFilesMetadataManager $metadataManager, private DisplayNameCache $displayNameCache, + private Database $cacheDb, ) { } @@ -54,4 +55,8 @@ public function getDisplayNameCache(): DisplayNameCache { public function getMetadataManager(): IFilesMetadataManager { return $this->metadataManager; } + + public function getCacheDb(): Database { + return $this->cacheDb; + } } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 365d28fc8c556..df72fdda4b1d8 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -68,7 +68,7 @@ public function selectTagUsage(): self { return $this; } - public function selectFileCache(string $alias = null, bool $joinExtendedCache = true) { + public function selectFileCache(string $alias = null, bool $joinExtendedCache = true): self { $name = $alias ?: 'filecache'; $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', 'storage_mtime', 'encrypted', 'etag', "$name.permissions", 'checksum', 'unencrypted_size') @@ -84,13 +84,13 @@ public function selectFileCache(string $alias = null, bool $joinExtendedCache = return $this; } - public function whereStorageId(int $storageId) { + public function whereStorageId(int $storageId): self { $this->andWhere($this->expr()->eq('storage', $this->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); return $this; } - public function whereFileId(int $fileId) { + public function whereFileId(int $fileId): self { $alias = $this->alias; if ($alias) { $alias .= '.'; @@ -103,13 +103,13 @@ public function whereFileId(int $fileId) { return $this; } - public function wherePath(string $path) { + public function wherePath(string $path): self { $this->andWhere($this->expr()->eq('path_hash', $this->createNamedParameter(md5($path)))); return $this; } - public function whereParent(int $parent) { + public function whereParent(int $parent): self { $alias = $this->alias; if ($alias) { $alias .= '.'; @@ -122,7 +122,7 @@ public function whereParent(int $parent) { return $this; } - public function whereParentInParameter(string $parameter) { + public function whereParentInParameter(string $parameter): self { $alias = $this->alias; if ($alias) { $alias .= '.'; diff --git a/lib/private/Files/Cache/Database.php b/lib/private/Files/Cache/Database.php new file mode 100644 index 0000000000000..37433673a883a --- /dev/null +++ b/lib/private/Files/Cache/Database.php @@ -0,0 +1,191 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Cache; + +use OC\DB\Exceptions\DbalException; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\ICache; +use OCP\ICacheFactory; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class Database { + private ICache $cache; + + public function __construct( + private IDBConnection $connection, // todo: multiple db connections for sharding (open connection lazy?) + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private IFilesMetadataManager $filesMetadataManager, + ICacheFactory $cacheFactory, + ) { + $this->cache = $cacheFactory->createLocal('storage_by_fileid'); + } + + private function connectionForStorageId(int $storage): IDBConnection { + return $this->databaseForShard($this->getShardForStorageId($storage)); + } + + public function queryForStorageId(int $storage): CacheQueryBuilder { + return $this->queryForShard($this->getShardForStorageId($storage)); + } + + private function databaseForShard(int $shard): IDBConnection { + return $this->connection; + } + + private function queryForShard(int $shard): CacheQueryBuilder { + // todo: select db based on shard + $query = new CacheQueryBuilder( + $this->databaseForShard($shard), + $this->systemConfig, + $this->logger, + $this->filesMetadataManager + ); + $query->allowTable('filecache'); + $query->allowTable('filecache_extended'); + $query->allowTable('files_metadata'); + return $query; + } + + public function getByFileId(int $fileId): ?CacheEntry { + $cachedStorage = $this->getCachedStorageIdForFileId($fileId); + if ($cachedStorage) { + $result = $this->queryByFileIdInShard($fileId, $this->getShardForStorageId($cachedStorage)); + if ($result && $result->getId() === $fileId) { + return $result; + } + } + + foreach ($this->getAllShards() as $shard) { + $result = $this->queryByFileIdInShard($fileId, $shard); + if ($result) { + $this->cache->set((string)$fileId, $result->getStorageId()); + return $result; + } + } + return null; + } + + private function queryByFileIdInShard(int $fileId, int $shard): ?CacheEntry { + $query = $this->queryForShard($shard)->selectFileCache(); + $query->andWhere($query->expr()->eq('fileid', $fileId, IQueryBuilder::PARAM_INT)); + + $row = $query->executeQuery()->fetchOne(); + return $row ? new CacheEntry($row) : null; + } + + /** + * @param list $fileIds + * @return array + */ + public function getByFileIds(array $fileIds): array { + $cachedStorages = $this->getCachedShardsForFileIds($fileIds); + + $foundItems = []; + foreach ($cachedStorages as $shard => $fileIdsForShard) { + $foundItems += $this->queryByFileIdsInShard($shard, $fileIdsForShard); + } + + $remainingIds = array_diff($fileIds, array_keys($foundItems)); + + if ($remainingIds) { + foreach ($this->getAllShards() as $shard) { + $items = $this->queryByFileIdsInShard($shard, $remainingIds); + $remainingIds = array_diff($remainingIds, array_keys($items)); + $foundItems += $items; + + if (count($remainingIds) === 0) { + break; + } + } + } + return array_values($foundItems); + } + + /** + * @param list $fileIds + * @return array + */ + public function queryByFileIdsInShard(int $shard, array $fileIds): array { + $query = $this->queryForShard($shard)->selectFileCache(); + $query->andWhere($query->expr()->eq('fileid', $fileIds, IQueryBuilder::PARAM_INT)); + $result = $query->executeQuery(); + $items = []; + while ($row = $result->fetchOne()) { + $items[(int)$row['fileid']] = new CacheEntry($row); + } + return $items; + } + + private function getCachedStorageIdForFileId(int $fileId): ?int { + $cached = $this->cache->get((string)$fileId); + return ($cached === null) ? null : (int)$cached; + } + + /** + * @param list $fileIds + * @return array> + */ + private function getCachedShardsForFileIds(array $fileIds): array { + $result = []; + foreach ($fileIds as $fileId) { + $storageId = $this->getCachedStorageIdForFileId($fileId); + if ($storageId) { + $shard = $this->getShardForStorageId($storageId); + $result[$shard][] = $fileId; + } + } + return $result; + } + + private function getShardForStorageId(int $storage): int { + return 0; + } + + /** + * @return list + */ + private function getAllShards(): array { + return [0]; + } + + public function beginTransaction(int $storageId): void { + $this->connectionForStorageId($storageId)->beginTransaction(); + } + + public function inTransaction(int $storageId): bool { + return $this->connectionForStorageId($storageId)->inTransaction(); + } + + public function commit(int $storageId): void { + $this->connectionForStorageId($storageId)->commit(); + } + + public function rollBack(int $storageId): void { + $this->connectionForStorageId($storageId)->rollBack(); + } +} diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index ba0f98f42f492..af7f7e19c10e0 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -62,7 +62,7 @@ public static function getGlobalCache() { } /** - * @param \OC\Files\Storage\Storage|string $storage + * @param IStorage|string $storage * @param bool $isAvailable * @throws \RuntimeException */ diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index 31410eea798e5..dbc837a92eca2 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -47,14 +47,14 @@ public function __construct(?ICache $cache, CacheDependencies $dependencies = nu $this->cache = $cache; if (!$dependencies && $cache instanceof Cache) { $this->mimetypeLoader = $cache->mimetypeLoader; - $this->connection = $cache->connection; + $this->cacheDb = $cache->cacheDb; $this->querySearchHelper = $cache->querySearchHelper; } else { if (!$dependencies) { $dependencies = Server::get(CacheDependencies::class); } $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); - $this->connection = $dependencies->getConnection(); + $this->cacheDb = $dependencies->getCacheDb(); $this->querySearchHelper = $dependencies->getQuerySearchHelper(); } } @@ -236,8 +236,8 @@ public function getStatus($file) { return $this->getCache()->getStatus($file); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** diff --git a/tests/lib/Files/Cache/CacheTest.php b/tests/lib/Files/Cache/CacheTest.php index 1b5c8e6e5ffcd..b3cb7b20490ea 100644 --- a/tests/lib/Files/Cache/CacheTest.php +++ b/tests/lib/Files/Cache/CacheTest.php @@ -507,17 +507,6 @@ public function testNonExisting() { $this->assertEquals([], $this->cache->getFolderContents('foo')); } - public function testGetById() { - $storageId = $this->storage->getId(); - $data = ['size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file']; - $id = $this->cache->put('foo', $data); - - if (strlen($storageId) > 64) { - $storageId = md5($storageId); - } - $this->assertEquals([$storageId, 'foo'], \OC\Files\Cache\Cache::getById($id)); - } - public function testStorageMTime() { $data = ['size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file']; $this->cache->put('foo', $data); @@ -535,15 +524,6 @@ public function testStorageMTime() { $this->assertEquals(25, $cachedData['mtime']); } - public function testLongId() { - $storage = new LongId([]); - $cache = $storage->getCache(); - $storageId = $storage->getId(); - $data = ['size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file']; - $id = $cache->put('foo', $data); - $this->assertEquals([md5($storageId), 'foo'], \OC\Files\Cache\Cache::getById($id)); - } - /** * this test show the bug resulting if we have no normalizer installed */