[wip] Filecache chunking

lib/composer/composer/autoload_classmap.php

@@ -1405,6 +1405,7 @@
  'OC\\Files\\Cache\\CacheDependencies' => $baseDir . '/lib/private/Files/Cache/CacheDependencies.php',
  'OC\\Files\\Cache\\CacheEntry' => $baseDir . '/lib/private/Files/Cache/CacheEntry.php',
  'OC\\Files\\Cache\\CacheQueryBuilder' => $baseDir . '/lib/private/Files/Cache/CacheQueryBuilder.php',
+ 'OC\\Files\\Cache\\Database' => $baseDir . '/lib/private/Files/Cache/Database.php',
  'OC\\Files\\Cache\\FailedCache' => $baseDir . '/lib/private/Files/Cache/FailedCache.php',
  'OC\\Files\\Cache\\FileAccess' => $baseDir . '/lib/private/Files/Cache/FileAccess.php',
  'OC\\Files\\Cache\\HomeCache' => $baseDir . '/lib/private/Files/Cache/HomeCache.php',

lib/composer/composer/autoload_static.php

@@ -1446,6 +1446,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
  'OC\\Files\\Cache\\CacheDependencies' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheDependencies.php',
  'OC\\Files\\Cache\\CacheEntry' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheEntry.php',
  'OC\\Files\\Cache\\CacheQueryBuilder' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheQueryBuilder.php',
+ 'OC\\Files\\Cache\\Database' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Database.php',
  'OC\\Files\\Cache\\FailedCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FailedCache.php',
  'OC\\Files\\Cache\\FileAccess' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FileAccess.php',
  'OC\\Files\\Cache\\HomeCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/HomeCache.php',

lib/private/DB/QueryBuilder/QueryBuilder.php

@@ -76,6 +76,19 @@
  /** @var string */
  protected $lastInsertedTable;
 
+ /**
+ * Tables that require special attention and thus can't be queried by default
+ *
+ * @var list<string>
+ */
+ protected array $shardedTables = [
+ 'filecache',
+ 'filecache_extended',
+ 'files_metadata'
+ ];
+
+ protected bool $sharded = false;
+
  /**
  * Initializes a new QueryBuilder.
  *
@@ -662,6 +675,17 @@
  return $this;
  }
 
+ /**
+ * @param string $table
+ * @return void
+ * @throws \Exception
+ */
+ private function checkTableAccess(string $table) {
+ if (in_array($table, $this->shardedTables) !== $this->sharded) {
+ throw new \Exception("current query isn't allowed to access the $table table");
+ }
+ }
+
  /**
  * Creates and adds a query root corresponding to the table identified by the
  * given alias, forming a cartesian product with any existing query roots.
@@ -678,6 +702,7 @@
  * @return $this This QueryBuilder instance.
  */
  public function from($from, $alias = null) {
+ $this->checkTableAccess($from);
  $this->queryBuilder->from(
  $this->getTableName($from),
  $this->quoteAlias($alias)
@@ -704,6 +729,8 @@
  * @return $this This QueryBuilder instance.
  */
  public function join($fromAlias, $join, $alias, $condition = null) {
+ $this->checkTableAccess($join);
+
  $this->queryBuilder->join(
  $this->quoteAlias($fromAlias),
  $this->getTableName($join),
@@ -732,6 +759,8 @@
  * @return $this This QueryBuilder instance.
  */
  public function innerJoin($fromAlias, $join, $alias, $condition = null) {
+ $this->checkTableAccess($join);
+
  $this->queryBuilder->innerJoin(
  $this->quoteAlias($fromAlias),
  $this->getTableName($join),
@@ -760,6 +789,8 @@
  * @return $this This QueryBuilder instance.
  */
  public function leftJoin($fromAlias, $join, $alias, $condition = null) {
+ $this->checkTableAccess($join);
+
  $this->queryBuilder->leftJoin(
  $this->quoteAlias($fromAlias),
  $this->getTableName($join),
@@ -788,6 +819,8 @@
  * @return $this This QueryBuilder instance.
  */
  public function rightJoin($fromAlias, $join, $alias, $condition = null) {
+ $this->checkTableAccess($join);
+
  $this->queryBuilder->rightJoin(
  $this->quoteAlias($fromAlias),
  $this->getTableName($join),
@@ -1337,4 +1370,20 @@
 
  return $this->helper->quoteColumnName($alias);
  }
+
+ public function escapeLikeParameter(string $parameter): string {
+ return $this->connection->escapeLikeParameter($parameter);
+ }
+
+ /**
+ * Mark the query as accessing the sharded tables
+ *
+ * Proper attention needs to be given to ensure that all requirements for accessing the sharded tables
+ *
+ * @param bool $sharded
+ * @return void
+ */
+ public function setSharded(bool $sharded): void {
+ $this->sharded = $sharded;
+ }
 }

lib/private/Files/Cache/Cache.php

@@ -62,7 +62,7 @@
 use OCP\Files\Search\ISearchQuery;
 use OCP\Files\Storage\IStorage;
 use OCP\FilesMetadata\IFilesMetadataManager;
-use OCP\IDBConnection;
+use OCP\Server;
 use OCP\Util;
 use Psr\Log\LoggerInterface;
 
@@ -88,7 +88,7 @@
  protected string $storageId;
  protected Storage $storageCache;
  protected IMimeTypeLoader$mimetypeLoader;
- protected IDBConnection $connection;
+ protected CacheDatabase $cacheDb;
  protected SystemConfig $systemConfig;
  protected LoggerInterface $logger;
  protected QuerySearchHelper $querySearchHelper;
@@ -106,25 +106,20 @@
  $this->storageId = md5($this->storageId);
  }
  if (!$dependencies) {
- $dependencies = \OC::$server->get(CacheDependencies::class);
+ $dependencies = Server::get(CacheDependencies::class);
  }
  $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection());
  $this->mimetypeLoader = $dependencies->getMimeTypeLoader();
- $this->connection = $dependencies->getConnection();
+ $this->cacheDb = $dependencies->getCacheDb();
  $this->systemConfig = $dependencies->getSystemConfig();
  $this->logger = $dependencies->getLogger();
  $this->querySearchHelper = $dependencies->getQuerySearchHelper();
  $this->eventDispatcher = $dependencies->getEventDispatcher();
  $this->metadataManager = $dependencies->getMetadataManager();
  }
 
- protected function getQueryBuilder() {
- return new CacheQueryBuilder(
- $this->connection,
- $this->systemConfig,
- $this->logger,
- $this->metadataManager,
- );
+ public function getQueryBuilder() {
+ return $this->cacheDb->queryForStorageId($this->getNumericStorageId());
  }
 
  public function getStorageCache(): Storage {
@@ -149,7 +144,7 @@
  public function get($file) {
  $query = $this->getQueryBuilder();
  $query->selectFileCache();
- $metadataQuery = $query->selectMetadata();
+ $metadataQuery = $query->selectMetadata($this->metadataManager);
 
  if (is_string($file) || $file == '') {
  // normalize file
@@ -236,7 +231,7 @@
  ->whereParent($fileId)
  ->orderBy('name', 'ASC');
 
- $metadataQuery = $query->selectMetadata();
+ $metadataQuery = $query->selectMetadata($this->metadataManager);
 
  $result = $query->execute();
  $files = $result->fetchAll();
@@ -305,7 +300,7 @@
  $values['storage'] = $storageId;
 
  try {
- $builder = $this->connection->getQueryBuilder();
+ $builder = $this->getQueryBuilder();
  $builder->insert('filecache');
 
  foreach ($values as $column => $value) {
@@ -333,9 +328,9 @@
  }
  } catch (UniqueConstraintViolationException $e) {
  // entry exists already
- if ($this->connection->inTransaction()) {
- $this->connection->commit();
- $this->connection->beginTransaction();
+ if ($this->cacheDb->inTransaction($this->getNumericStorageId())) {
+ $this->cacheDb->commit($this->getNumericStorageId());
+ $this->cacheDb->beginTransaction($this->getNumericStorageId());
  }
  }
 
@@ -699,7 +694,7 @@
  if ($sourceData['mimetype'] === 'httpd/unix-directory') {
  //update all child entries
  $sourceLength = mb_strlen($sourcePath);
- $query = $this->connection->getQueryBuilder();
+ $query = $this->getQueryBuilder();
 
  $fun = $query->func();
  $newPathFunction = $fun->concat(
@@ -711,7 +706,7 @@
  ->set('path_hash', $fun->md5($newPathFunction))
  ->set('path', $newPathFunction)
  ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT)))
- ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%')));
+ ->andWhere($query->expr()->like('path', $query->createNamedParameter($query->escapeLikeParameter($sourcePath) . '/%')));
 
  // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
@@ -723,26 +718,26 @@
  $retryLimit = 4;
  for ($i = 1; $i <= $retryLimit; $i++) {
  try {
- $this->connection->beginTransaction();
+ $this->cacheDb->beginTransaction();
  $query->executeStatement();
  break;
  } catch (\OC\DatabaseException $e) {
- $this->connection->rollBack();
+ $this->cacheDb->rollBack($this->getNumericStorageId());
  throw $e;
  } catch (RetryableException $e) {
  // Simply throw if we already retried 4 times.
  if ($i === $retryLimit) {
  throw $e;
  }
 
- $this->connection->rollBack();
+ $this->cacheDb->rollBack();
 
  // Sleep a bit to give some time to the other transaction to finish.
  usleep(100 * 1000 * $i);
  }
  }
  } else {
- $this->connection->beginTransaction();
+ $this->cacheDb->beginTransaction();
  }
 
  $query = $this->getQueryBuilder();
@@ -761,7 +756,7 @@
 
  $query->execute();
 
- $this->connection->commit();
+ $this->cacheDb->commit($this->getNumericStorageId());
 
  if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) {
  $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()));
@@ -787,7 +782,7 @@
  ->whereStorageId($this->getNumericStorageId());
  $query->execute();
 
- $query = $this->connection->getQueryBuilder();
+ $query = $this->getQueryBuilder();
  $query->delete('storages')
  ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId)));
  $query->execute();
@@ -861,8 +856,8 @@
  return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
  }
 
- public function searchQuery(ISearchQuery $searchQuery) {
- return current($this->querySearchHelper->searchInCaches($searchQuery, [$this]));
+ public function searchQuery(ISearchQuery $query) {
+ return current($this->querySearchHelper->searchInCaches($query, [$this]));
  }
 
  /**
@@ -1099,27 +1094,7 @@
  * @deprecated use getPathById() instead
  */
  public static function getById($id) {
- $query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
- $query->select('path', 'storage')
- ->from('filecache')
- ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
-
- $result = $query->execute();
- $row = $result->fetch();
- $result->closeCursor();
-
- if ($row) {
- $numericId = $row['storage'];
- $path = $row['path'];
- } else {
- return null;
- }
-
- if ($id = Storage::getStorageId($numericId)) {
- return [$id, $path];
- } else {
- return null;
- }
+ throw new \Exception("Cache::getById has been removed");
  }
 
  /**