diff --git a/lib/BackgroundJobs/IndexerJob.php b/lib/BackgroundJobs/IndexerJob.php index cd1c00ca..58a24a53 100644 --- a/lib/BackgroundJobs/IndexerJob.php +++ b/lib/BackgroundJobs/IndexerJob.php @@ -47,6 +47,7 @@ class IndexerJob extends TimedJob { public const DEFAULT_JOB_INTERVAL = 30 * 60; public const DEFAULT_MAX_INDEXING_TIME = 30 * 60; + private const INDEX_COMPLETION_THRESHOLD = 0.02; // 2% // Assuming a backend capacity of 50 files per minute we can send 1500 files in half an hour // Specifying a higher number here will still be overruled by the max indexing time @@ -325,15 +326,21 @@ private function setInitialIndexCompletion(): void { } try { $queuedNewFilesCount = $this->queue->countNewFiles(); + $eligibleFilesCount = $this->storageService->countFiles(); + // if the new files in the queue are less than 2% of the total eligible files, we consider the + // initial indexing complete this allows for some margin of error in case some files were + // added while we were indexing but still ensures that we have indexed the vast majority of + // files at least once + $thresholdCount = (float)$eligibleFilesCount * self::INDEX_COMPLETION_THRESHOLD; } catch (\OCP\DB\Exception $e) { - $this->logger->warning('Could not count queued new files', ['exception' => $e]); + $this->logger->warning('Could not count queued new files or total eligible files', ['exception' => $e]); return; } $crawlJobCount = $this->getJobCount(StorageCrawlJob::class); // if any storage crawler jobs are still running or there are still new files in the queue, // we are still indexing files that were never indexed before. - if ($crawlJobCount > 0 || $queuedNewFilesCount > 0) { + if ($crawlJobCount > 0 || $queuedNewFilesCount > $thresholdCount) { return; } diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index fbc1aaab..b9372591 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -13,12 +13,11 @@ use OC\Files\Cache\CacheQueryBuilder; use OCA\ContextChat\AppInfo\Application; use OCA\ContextChat\Logger; -use OCP\DB\Exception; +use OCP\DB\Exception as DBException; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IFileAccess; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; -use OCP\Files\File; use OCP\Files\Folder; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; @@ -51,7 +50,7 @@ public function __construct( } /** - * @throws Exception + * @throws DBException */ public function countFiles(): int { $totalCount = 0; @@ -75,7 +74,7 @@ public function countFilesInMount(int $storageId, int $rootId): int { /** @var array{path:string}|false $root */ $root = $result->fetch(); $result->closeCursor(); - } catch (Exception $e) { + } catch (DBException $e) { $this->logger->error('Could not fetch storage root', ['exception' => $e]); return 0; } @@ -115,7 +114,7 @@ public function countFilesInMount(int $storageId, int $rootId): int { ->andWhere($qb->expr()->lte('filecache.size', $qb->createNamedParameter(Application::CC_MAX_SIZE, IQueryBuilder::PARAM_INT))) ->andWhere($qb->expr()->gt('filecache.size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); $result = $qb->executeQuery(); - } catch (Exception $e) { + } catch (DBException $e) { $this->logger->error('Could not count files in mount: storage=' . $storageId . ' root=' . $rootId, ['exception' => $e]); return 0; } @@ -136,7 +135,7 @@ private function isFileAccessAvailable(): bool { /** * @return \Generator - * @throws \OCP\DB\Exception + * @throws DBException */ public function getMounts(): \Generator { if (!$this->isFileAccessAvailable()) { @@ -148,7 +147,7 @@ public function getMounts(): \Generator { /** * @return \Generator - * @throws \OCP\DB\Exception + * @throws DBException */ private function getMountsOld(): \Generator { $qb = $this->db->getQueryBuilder(); @@ -181,7 +180,7 @@ private function getMountsOld(): \Generator { if ($root !== false) { $overrideRoot = (int)$root['fileid']; } - } catch (Exception $e) { + } catch (DBException $e) { $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); continue; } @@ -240,7 +239,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId /** @var array{path:string}|false $root */ $root = $result->fetch(); $result->closeCursor(); - } catch (Exception $e) { + } catch (DBException $e) { $this->logger->error('Could not fetch storage root', ['exception' => $e]); return; } @@ -280,7 +279,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId } $files = $qb->orderBy('filecache.fileid', 'ASC') ->executeQuery(); - } catch (Exception $e) { + } catch (DBException $e) { $this->logger->error('Could not fetch files', ['exception' => $e]); return; }