Skip to content

perf: Partition preview migration job #64515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CarlSchwan wants to merge 1 commit into
base: master
Choose a base branch
from
+67 −61
Open

perf: Partition preview migration job #64515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 53 additions & 22 deletions core/BackgroundJobs/PreviewMigrationJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

use OC\Preview\PreviewMigrationService;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\FileInfo;
use OCP\Files\IRootFolder;
Expand All @@ -20,64 +21,79 @@
use Psr\Log\LoggerInterface;

class PreviewMigrationJob extends TimedJob {
public const int PARTITIONS = 16;
private string $previewRootPath;

public function __construct(
ITimeFactory $time,
private readonly IAppConfig $appConfig,
private readonly IConfig $config,
IConfig $config,
private readonly IRootFolder $rootFolder,
private readonly PreviewMigrationService $migrationService,
private readonly IJobList $jobList,
private readonly LoggerInterface $logger,
) {
parent::__construct($time);

$this->setTimeSensitivity(self::TIME_INSENSITIVE);
$this->setInterval(24 * 60 * 60);
$this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/';
$this->setInterval(24 * 60);
$this->previewRootPath = 'appdata_' . $config->getSystemValueString('instanceid') . '/preview/';
}

#[Override]
protected function run(mixed $argument): void {
if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
$this->jobList->removeById($this->getId());
return;
}

$partition = (int)($argument['partition'] ?? 0);
if ($partition < 0 || $partition >= self::PARTITIONS) {
$this->jobList->removeById($this->getId());
return;
}

if (!$this->runPartition($partition)) {
return;
}

$this->appConfig->setValueBool('core', 'previewMigrationPartition' . $partition, true);
$this->jobList->removeById($this->getId());
for ($i = 0; $i < self::PARTITIONS; $i++) {
if (!$this->appConfig->getValueBool('core', 'previewMigrationPartition' . $i, false)) {
return;
}
}

$this->appConfig->setValueBool('core', 'previewMovedDone', true);
}

private function runPartition(int $partition): bool {
$storage = $this->rootFolder->getMountPoint()->getStorage();
if ($storage === null) {
$this->logger->warning('Preview migration skipped: the root mount point has no storage.');
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
return;
return true;
}

$cache = $storage->getCache();
$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
if ($previewRootId === -1) {
// No previews were ever generated, or the storage config no longer
// matches the one the filecache data was recorded under.
$this->logger->warning('Preview migration skipped: no preview root found at "{path}" on storage "{storageId}".', [
'path' => $this->previewRootPath,
'storageId' => $storage->getId(),
]);
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
return;
return true;
}

$startTime = time();

// Walk the preview folder tree via the `parent` column, which is indexed on
// every supported database platform.
//
// Depth from the preview root tells us which structure a leaf folder holds:
// - depth 1: legacy flat structure, e.g. preview/<fileid>/<size>.png
// - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g/<fileid>/<size>.png
$foldersToVisit = [[$previewRootId, '', 0]];

while ($foldersToVisit !== []) {
[$folderId, $folderName, $depth] = array_pop($foldersToVisit);

// Collect the actual preview files here so migrateFileId() doesn't need to
// list this folder's contents a second time.
if ($depth === 1 && !$this->belongsToPartition($folderName, $partition)) {
continue;
}

$previewEntries = [];
foreach ($cache->getFolderContentsById($folderId) as $entry) {
if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
Expand All @@ -99,12 +115,27 @@ protected function run(mixed $argument): void {
]);
}

// Stop if execution time is more than one hour.
if (time() - $startTime > 3600) {
return;
return false;
}
}

$this->appConfig->setValueBool('core', 'previewMovedDone', true);
return true;
}

private function belongsToPartition(string $folderName, int $partition): bool {
if ($partition < 0 || $partition >= self::PARTITIONS) {
return false;
}

if (ctype_digit($folderName)) {
return ((int)$folderName % self::PARTITIONS) === $partition;
}

if (strlen($folderName) === 1 && ctype_xdigit($folderName)) {
return (hexdec($folderName) % self::PARTITIONS) === $partition;
}

return $partition === 0;
}
}
11 changes: 10 additions & 1 deletion lib/private/Repair/AddMovePreviewJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@

use OC\Core\BackgroundJobs\PreviewMigrationJob;
use OCP\BackgroundJob\IJobList;
use OCP\IAppConfig;
use OCP\Migration\IOutput;
use OCP\Migration\IRepairStep;
use Override;

class AddMovePreviewJob implements IRepairStep {
public function __construct(
private readonly IJobList $jobList,
private readonly IAppConfig $appConfig,
) {
}

Expand All @@ -28,6 +30,13 @@ public function getName(): string {

#[Override]
public function run(IOutput $output): void {
$this->jobList->add(PreviewMigrationJob::class);
// Remove the unpartitioned job registered by older server versions.
$this->jobList->remove(PreviewMigrationJob::class);
for ($partition = 0; $partition < PreviewMigrationJob::PARTITIONS; $partition++) {
$this->appConfig->setValueBool('core', 'previewMigrationPartition' . $partition, false);
$this->jobList->add(PreviewMigrationJob::class, [
'partition' => $partition,
]);
}
}
}
2 changes: 0 additions & 2 deletions lib/private/Setup.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
use OC\Core\BackgroundJobs\CleanupBackgroundJobsJob;
use OC\Core\BackgroundJobs\ExpirePreviewsJob;
use OC\Core\BackgroundJobs\GenerateMetadataJob;
use OC\Core\BackgroundJobs\PreviewMigrationJob;
use OC\Log\Rotate;
use OC\Preview\BackgroundCleanupJob;
use OC\Setup\AbstractDatabase;
Expand Down Expand Up @@ -533,7 +532,6 @@ public static function installBackgroundJobs(): void {
$jobList->add(CleanupDeletedUsers::class);
$jobList->add(CleanupLoginTokens::class);
$jobList->add(GenerateMetadataJob::class);
$jobList->add(PreviewMigrationJob::class);
$jobList->add(ExpirePreviewsJob::class);
$jobList->add(CleanupBackgroundJobsJob::class);
}
Expand Down
40 changes: 4 additions & 36 deletions tests/lib/Preview/PreviewMigrationJobTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
use OC\Preview\PreviewService;
use OC\Preview\Storage\StorageFactory;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\Files\AppData\IAppDataFactory;
use OCP\Files\IAppData;
use OCP\Files\IMimeTypeDetector;
Expand Down Expand Up @@ -143,6 +144,7 @@ private function createJob(): PreviewMigrationJob {
$this->storageFactory,
Server::get(IAppDataFactory::class),
),
Server::get(IJobList::class),
$this->logger,
);
}
Expand Down Expand Up @@ -258,24 +260,7 @@ public function testMigrationPath(): void {
$this->assertEquals(2, count($folder->getDirectoryListing()));
$this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5))));

$job = new PreviewMigrationJob(
Server::get(ITimeFactory::class),
$this->appConfig,
$this->config,
Server::get(IRootFolder::class),
new PreviewMigrationService(
$this->config,
Server::get(IRootFolder::class),
$this->logger,
$this->mimeTypeDetector,
$this->mimeTypeLoader,
Server::get(IDBConnection::class),
$this->previewMapper,
$this->storageFactory,
Server::get(IAppDataFactory::class),
),
$this->logger,
);
$job = $this->createJob();
$this->invokePrivate($job, 'run', [[]]);
$this->assertEquals(0, count($this->previewAppData->getDirectoryListing()));
$this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5))));
Expand Down Expand Up @@ -303,24 +288,7 @@ public function testMigrationPathWithVersion(): void {
$this->assertEquals(9, count($folder->getDirectoryListing()));
$this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5))));

$job = new PreviewMigrationJob(
Server::get(ITimeFactory::class),
$this->appConfig,
$this->config,
Server::get(IRootFolder::class),
new PreviewMigrationService(
$this->config,
Server::get(IRootFolder::class),
$this->logger,
$this->mimeTypeDetector,
$this->mimeTypeLoader,
Server::get(IDBConnection::class),
$this->previewMapper,
$this->storageFactory,
Server::get(IAppDataFactory::class),
),
$this->logger,
);
$job = $this->createJob();
$this->invokePrivate($job, 'run', [[]]);
$previews = iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5));
$this->assertEquals(9, count($previews));
Expand Down
Loading