Skip to content

fix: use 2 percent index completion threshold #177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kyteinsky merged 2 commits into from
Sep 22, 2025
Merged

fix: use 2 percent index completion threshold #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f1340f
fix: use 2 percent index completion threshold
kyteinsky Sep 22, 2025
6abdcc1
chore: alias OCP\DB\Exception as DBException
kyteinsky Sep 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions lib/BackgroundJobs/IndexerJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class IndexerJob extends TimedJob {

public const DEFAULT_JOB_INTERVAL = 30 * 60;
public const DEFAULT_MAX_INDEXING_TIME = 30 * 60;
private const INDEX_COMPLETION_THRESHOLD = 0.02; // 2%

// Assuming a backend capacity of 50 files per minute we can send 1500 files in half an hour
// Specifying a higher number here will still be overruled by the max indexing time
Expand Down Expand Up @@ -325,15 +326,21 @@ private function setInitialIndexCompletion(): void {
}
try {
$queuedNewFilesCount = $this->queue->countNewFiles();
$eligibleFilesCount = $this->storageService->countFiles();
// if the new files in the queue are less than 2% of the total eligible files, we consider the
// initial indexing complete this allows for some margin of error in case some files were
// added while we were indexing but still ensures that we have indexed the vast majority of
// files at least once
$thresholdCount = (float)$eligibleFilesCount * self::INDEX_COMPLETION_THRESHOLD;
} catch (\OCP\DB\Exception $e) {
$this->logger->warning('Could not count queued new files', ['exception' => $e]);
$this->logger->warning('Could not count queued new files or total eligible files', ['exception' => $e]);
return;
}
Comment thread
kyteinsky marked this conversation as resolved.
$crawlJobCount = $this->getJobCount(StorageCrawlJob::class);

// if any storage crawler jobs are still running or there are still new files in the queue,
// we are still indexing files that were never indexed before.
if ($crawlJobCount > 0 || $queuedNewFilesCount > 0) {
if ($crawlJobCount > 0 || $queuedNewFilesCount > $thresholdCount) {
return;
}

Expand Down
19 changes: 9 additions & 10 deletions lib/Service/StorageService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@
use OC\Files\Cache\CacheQueryBuilder;
use OCA\ContextChat\AppInfo\Application;
use OCA\ContextChat\Logger;
use OCP\DB\Exception;
use OCP\DB\Exception as DBException;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\IFileAccess;
use OCP\Files\Config\ICachedMountInfo;
use OCP\Files\Config\IUserMountCache;
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\IRootFolder;
Expand Down Expand Up @@ -51,7 +50,7 @@ public function __construct(
}

/**
* @throws Exception
* @throws DBException
*/
public function countFiles(): int {
$totalCount = 0;
Expand All @@ -75,7 +74,7 @@ public function countFilesInMount(int $storageId, int $rootId): int {
/** @var array{path:string}|false $root */
$root = $result->fetch();
$result->closeCursor();
} catch (Exception $e) {
} catch (DBException $e) {
$this->logger->error('Could not fetch storage root', ['exception' => $e]);
return 0;
}
Expand Down Expand Up @@ -115,7 +114,7 @@ public function countFilesInMount(int $storageId, int $rootId): int {
->andWhere($qb->expr()->lte('filecache.size', $qb->createNamedParameter(Application::CC_MAX_SIZE, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->gt('filecache.size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
$result = $qb->executeQuery();
} catch (Exception $e) {
} catch (DBException $e) {
$this->logger->error('Could not count files in mount: storage=' . $storageId . ' root=' . $rootId, ['exception' => $e]);
return 0;
}
Expand All @@ -136,7 +135,7 @@ private function isFileAccessAvailable(): bool {

/**
* @return \Generator<array{root_id: int, overridden_root: int, storage_id: int}>
* @throws \OCP\DB\Exception
* @throws DBException
*/
public function getMounts(): \Generator {
if (!$this->isFileAccessAvailable()) {
Expand All @@ -148,7 +147,7 @@ public function getMounts(): \Generator {

/**
* @return \Generator<array{root_id: int, overridden_root: int, storage_id: int}>
* @throws \OCP\DB\Exception
* @throws DBException
*/
private function getMountsOld(): \Generator {
$qb = $this->db->getQueryBuilder();
Expand Down Expand Up @@ -181,7 +180,7 @@ private function getMountsOld(): \Generator {
if ($root !== false) {
$overrideRoot = (int)$root['fileid'];
}
} catch (Exception $e) {
} catch (DBException $e) {
$this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]);
continue;
}
Expand Down Expand Up @@ -240,7 +239,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId
/** @var array{path:string}|false $root */
$root = $result->fetch();
$result->closeCursor();
} catch (Exception $e) {
} catch (DBException $e) {
$this->logger->error('Could not fetch storage root', ['exception' => $e]);
return;
}
Expand Down Expand Up @@ -280,7 +279,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId
}
$files = $qb->orderBy('filecache.fileid', 'ASC')
->executeQuery();
} catch (Exception $e) {
} catch (DBException $e) {
$this->logger->error('Could not fetch files', ['exception' => $e]);
return;
}
Expand Down
Loading