From afc2860ba6840e4ba4b73dbdfe43a36599adddd8 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 18 Sep 2026 11:27:43 +0200 Subject: [PATCH] perf: Partition preview migration job Use 16 partitions Assisted-by: ClaudeCode:claude-opus-5 Signed-off-by: Carl Schwan --- core/BackgroundJobs/PreviewMigrationJob.php | 75 +++++++++++++------ lib/private/Repair/AddMovePreviewJob.php | 11 ++- lib/private/Setup.php | 2 - tests/lib/Preview/PreviewMigrationJobTest.php | 40 +--------- 4 files changed, 67 insertions(+), 61 deletions(-) diff --git a/core/BackgroundJobs/PreviewMigrationJob.php b/core/BackgroundJobs/PreviewMigrationJob.php index 45f71e561244f..77aebff790d8c 100644 --- a/core/BackgroundJobs/PreviewMigrationJob.php +++ b/core/BackgroundJobs/PreviewMigrationJob.php @@ -11,6 +11,7 @@ use OC\Preview\PreviewMigrationService; use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\TimedJob; use OCP\Files\FileInfo; use OCP\Files\IRootFolder; @@ -20,64 +21,79 @@ use Psr\Log\LoggerInterface; class PreviewMigrationJob extends TimedJob { + public const int PARTITIONS = 16; private string $previewRootPath; public function __construct( ITimeFactory $time, private readonly IAppConfig $appConfig, - private readonly IConfig $config, + IConfig $config, private readonly IRootFolder $rootFolder, private readonly PreviewMigrationService $migrationService, + private readonly IJobList $jobList, private readonly LoggerInterface $logger, ) { parent::__construct($time); - $this->setTimeSensitivity(self::TIME_INSENSITIVE); - $this->setInterval(24 * 60 * 60); - $this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/'; + $this->setInterval(24 * 60); + $this->previewRootPath = 'appdata_' . $config->getSystemValueString('instanceid') . '/preview/'; } #[Override] protected function run(mixed $argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { + $this->jobList->removeById($this->getId()); + return; + } + + $partition = (int)($argument['partition'] ?? 0); + if ($partition < 0 || $partition >= self::PARTITIONS) { + $this->jobList->removeById($this->getId()); return; } + if (!$this->runPartition($partition)) { + return; + } + + $this->appConfig->setValueBool('core', 'previewMigrationPartition' . $partition, true); + $this->jobList->removeById($this->getId()); + for ($i = 0; $i < self::PARTITIONS; $i++) { + if (!$this->appConfig->getValueBool('core', 'previewMigrationPartition' . $i, false)) { + return; + } + } + + $this->appConfig->setValueBool('core', 'previewMovedDone', true); + } + + private function runPartition(int $partition): bool { $storage = $this->rootFolder->getMountPoint()->getStorage(); if ($storage === null) { $this->logger->warning('Preview migration skipped: the root mount point has no storage.'); - $this->appConfig->setValueBool('core', 'previewMovedDone', true); - return; + return true; } $cache = $storage->getCache(); $previewRootId = $cache->getId(rtrim($this->previewRootPath, '/')); if ($previewRootId === -1) { - // No previews were ever generated, or the storage config no longer - // matches the one the filecache data was recorded under. $this->logger->warning('Preview migration skipped: no preview root found at "{path}" on storage "{storageId}".', [ 'path' => $this->previewRootPath, 'storageId' => $storage->getId(), ]); - $this->appConfig->setValueBool('core', 'previewMovedDone', true); - return; + return true; } $startTime = time(); - - // Walk the preview folder tree via the `parent` column, which is indexed on - // every supported database platform. - // - // Depth from the preview root tells us which structure a leaf folder holds: - // - depth 1: legacy flat structure, e.g. preview//.png - // - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g//.png $foldersToVisit = [[$previewRootId, '', 0]]; while ($foldersToVisit !== []) { [$folderId, $folderName, $depth] = array_pop($foldersToVisit); - // Collect the actual preview files here so migrateFileId() doesn't need to - // list this folder's contents a second time. + if ($depth === 1 && !$this->belongsToPartition($folderName, $partition)) { + continue; + } + $previewEntries = []; foreach ($cache->getFolderContentsById($folderId) as $entry) { if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { @@ -99,12 +115,27 @@ protected function run(mixed $argument): void { ]); } - // Stop if execution time is more than one hour. if (time() - $startTime > 3600) { - return; + return false; } } - $this->appConfig->setValueBool('core', 'previewMovedDone', true); + return true; + } + + private function belongsToPartition(string $folderName, int $partition): bool { + if ($partition < 0 || $partition >= self::PARTITIONS) { + return false; + } + + if (ctype_digit($folderName)) { + return ((int)$folderName % self::PARTITIONS) === $partition; + } + + if (strlen($folderName) === 1 && ctype_xdigit($folderName)) { + return (hexdec($folderName) % self::PARTITIONS) === $partition; + } + + return $partition === 0; } } diff --git a/lib/private/Repair/AddMovePreviewJob.php b/lib/private/Repair/AddMovePreviewJob.php index bf89464583b87..3251f0978f1f7 100644 --- a/lib/private/Repair/AddMovePreviewJob.php +++ b/lib/private/Repair/AddMovePreviewJob.php @@ -11,6 +11,7 @@ use OC\Core\BackgroundJobs\PreviewMigrationJob; use OCP\BackgroundJob\IJobList; +use OCP\IAppConfig; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; use Override; @@ -18,6 +19,7 @@ class AddMovePreviewJob implements IRepairStep { public function __construct( private readonly IJobList $jobList, + private readonly IAppConfig $appConfig, ) { } @@ -28,6 +30,13 @@ public function getName(): string { #[Override] public function run(IOutput $output): void { - $this->jobList->add(PreviewMigrationJob::class); + // Remove the unpartitioned job registered by older server versions. + $this->jobList->remove(PreviewMigrationJob::class); + for ($partition = 0; $partition < PreviewMigrationJob::PARTITIONS; $partition++) { + $this->appConfig->setValueBool('core', 'previewMigrationPartition' . $partition, false); + $this->jobList->add(PreviewMigrationJob::class, [ + 'partition' => $partition, + ]); + } } } diff --git a/lib/private/Setup.php b/lib/private/Setup.php index 8033bd520998b..411f7231afa5a 100644 --- a/lib/private/Setup.php +++ b/lib/private/Setup.php @@ -19,7 +19,6 @@ use OC\Core\BackgroundJobs\CleanupBackgroundJobsJob; use OC\Core\BackgroundJobs\ExpirePreviewsJob; use OC\Core\BackgroundJobs\GenerateMetadataJob; -use OC\Core\BackgroundJobs\PreviewMigrationJob; use OC\Log\Rotate; use OC\Preview\BackgroundCleanupJob; use OC\Setup\AbstractDatabase; @@ -533,7 +532,6 @@ public static function installBackgroundJobs(): void { $jobList->add(CleanupDeletedUsers::class); $jobList->add(CleanupLoginTokens::class); $jobList->add(GenerateMetadataJob::class); - $jobList->add(PreviewMigrationJob::class); $jobList->add(ExpirePreviewsJob::class); $jobList->add(CleanupBackgroundJobsJob::class); } diff --git a/tests/lib/Preview/PreviewMigrationJobTest.php b/tests/lib/Preview/PreviewMigrationJobTest.php index dcd57464f0d1e..73c585db50dd2 100644 --- a/tests/lib/Preview/PreviewMigrationJobTest.php +++ b/tests/lib/Preview/PreviewMigrationJobTest.php @@ -16,6 +16,7 @@ use OC\Preview\PreviewService; use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; use OCP\Files\IMimeTypeDetector; @@ -143,6 +144,7 @@ private function createJob(): PreviewMigrationJob { $this->storageFactory, Server::get(IAppDataFactory::class), ), + Server::get(IJobList::class), $this->logger, ); } @@ -258,24 +260,7 @@ public function testMigrationPath(): void { $this->assertEquals(2, count($folder->getDirectoryListing())); $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); - $job = new PreviewMigrationJob( - Server::get(ITimeFactory::class), - $this->appConfig, - $this->config, - Server::get(IRootFolder::class), - new PreviewMigrationService( - $this->config, - Server::get(IRootFolder::class), - $this->logger, - $this->mimeTypeDetector, - $this->mimeTypeLoader, - Server::get(IDBConnection::class), - $this->previewMapper, - $this->storageFactory, - Server::get(IAppDataFactory::class), - ), - $this->logger, - ); + $job = $this->createJob(); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); @@ -303,24 +288,7 @@ public function testMigrationPathWithVersion(): void { $this->assertEquals(9, count($folder->getDirectoryListing())); $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); - $job = new PreviewMigrationJob( - Server::get(ITimeFactory::class), - $this->appConfig, - $this->config, - Server::get(IRootFolder::class), - new PreviewMigrationService( - $this->config, - Server::get(IRootFolder::class), - $this->logger, - $this->mimeTypeDetector, - $this->mimeTypeLoader, - Server::get(IDBConnection::class), - $this->previewMapper, - $this->storageFactory, - Server::get(IAppDataFactory::class), - ), - $this->logger, - ); + $job = $this->createJob(); $this->invokePrivate($job, 'run', [[]]); $previews = iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)); $this->assertEquals(9, count($previews));