Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ jobs:
# See version matrix @ https://typo3.org/cms/roadmap
- { php: 8.2 }
- { php: 8.3 }
- { php: 8.4 }
- { php: 8.5 }

env: ${{ matrix.env }}

Expand Down
80 changes: 35 additions & 45 deletions Classes/Command/CleanupIndexCommand.php
Original file line number Diff line number Diff line change
@@ -1,22 +1,30 @@
<?php

declare(strict_types=1);

namespace Networkteam\SolrCommands\Command;

use ApacheSolrForTypo3\Solr\ConnectionManager;
use ApacheSolrForTypo3\Solr\Domain\Search\Query\Query;
use ApacheSolrForTypo3\Solr\Domain\Site\SiteRepository;
use ApacheSolrForTypo3\Solr\System\Solr\Document\Document;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\GeneralUtility;

class CleanupIndexCommand extends Command
{
public function configure()
public function __construct(
private readonly SiteRepository $siteRepository,
private readonly ConnectionManager $connectionManager,
private readonly ConnectionPool $connectionPool,
) {
parent::__construct();
}

protected function configure(): void
{
$this->setDescription('Delete documents in index which are not in the index queue');
}
Expand All @@ -25,27 +33,20 @@ protected function execute(InputInterface $input, OutputInterface $output): int
{
$rootPageItemTypeMap = $this->getRootPageItemTypeMap();

/** @var SiteRepository $siteRepository */
$siteRepository = GeneralUtility::makeInstance(SiteRepository::class);

foreach($rootPageItemTypeMap as $item) {
foreach ($rootPageItemTypeMap as $item) {
$rootPage = $item['root'];
$itemType = $item['item_type'];

$site = $siteRepository->getSiteByPageId($rootPage);
$site = $this->siteRepository->getSiteByPageId($rootPage);
if (!$site) {
continue;
}
$indexQueueItems = $this->getItemsByRootPageAndType($rootPage, $itemType);

$solrServers = GeneralUtility::makeInstance(ConnectionManager::class)->getConnectionsBySite($site);
$indexQueueItems = $this->getItemsByRootPageAndType($rootPage, $itemType);
$solrServers = $this->connectionManager->getConnectionsBySite($site);

/** @var Query $query */
$query = GeneralUtility::makeInstance(Query::class);
$query->setQuery(sprintf('type:%s AND siteHash:%s',
$itemType,
$site->getSiteHash(),
));
$query->setQuery(sprintf('type:%s AND siteHash:%s', $itemType, $site->getSiteHash()));
$query->setFields(['uid']);

foreach ($solrServers as $server) {
Expand All @@ -57,16 +58,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$documentUidsToDelete = [];
$i = 0;
$pages = ceil($numFound / 1000);
$pages = (int)ceil($numFound / 1000);
while ($i <= $pages) {
$query->addParam('start', $i * 1000);
$query->addParam('rows', 1000);
$response = $server->getReadService()->search($query);

/** @var Document[]|null $documents */
$documents = $response->getParsedData()?->response?->docs;

foreach($documents ?? [] as $document) {
$documents = $response->getParsedData()?->response?->docs ?? [];
foreach ($documents as $document) {
$uid = $document->getFields()['uid'];
if (!in_array($uid, $indexQueueItems)) {
$documentUidsToDelete[] = $uid;
Expand All @@ -77,55 +76,46 @@ protected function execute(InputInterface $input, OutputInterface $output): int
}

foreach ($documentUidsToDelete as $uid) {
$deleteQuery = sprintf('type:%s AND siteHash:%s AND uid:%s',
$itemType,
$site->getSiteHash(),
$uid
);

if ($output->isVerbose()) {
$output->writeln(sprintf('Deleting uid:%d (type: %s, root page: %d)', $uid, $itemType, $rootPage));
}

$server->getWriteService()->deleteByQuery($deleteQuery);
$server->getWriteService()->deleteByQuery(sprintf(
'type:%s AND siteHash:%s AND uid:%s',
$itemType,
$site->getSiteHash(),
$uid,
));
}

$server->getWriteService()->commit();
}
}

return 0;
return Command::SUCCESS;
}

protected function getRootPageItemTypeMap(): array
private function getRootPageItemTypeMap(): array
{
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('tx_solr_indexqueue_item');
$map = $queryBuilder
return $this->connectionPool
->getQueryBuilderForTable('tx_solr_indexqueue_item')
->select('root', 'item_type')
->from('tx_solr_indexqueue_item')
->groupBy('root', 'item_type')
->executeQuery()
->fetchAllAssociative();

return $map;
}

protected function getItemsByRootPageAndType(int $rootPage, string $type): array
private function getItemsByRootPageAndType(int $rootPage, string $type): array
{
/** @var QueryBuilder $queryBuilder */
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('tx_solr_indexqueue_item');
$query = $queryBuilder
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('tx_solr_indexqueue_item');
return $queryBuilder
->select('item_uid')
->from('tx_solr_indexqueue_item')
->where(
$queryBuilder->expr()->eq('root', $queryBuilder->createNamedParameter($rootPage, Connection::PARAM_INT)),
$queryBuilder->expr()->eq('item_type', $queryBuilder->createNamedParameter($type))
$queryBuilder->expr()->eq('item_type', $queryBuilder->createNamedParameter($type)),
)
->executeQuery();

$items = $query->fetchFirstColumn();
return $items;
->executeQuery()
->fetchFirstColumn();
}
}
28 changes: 16 additions & 12 deletions Classes/Command/ClearIndexCommand.php
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<?php

declare(strict_types=1);

namespace Networkteam\SolrCommands\Command;

use ApacheSolrForTypo3\Solr\ConnectionManager;
Expand All @@ -9,30 +11,32 @@
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;

class ClearIndexCommand extends Command
{
protected function configure()
public function __construct(
private readonly SiteRepository $siteRepository,
private readonly ConnectionManager $connectionManager,
) {
parent::__construct();
}

protected function configure(): void
{
$this->setDescription('Clear index')
->addArgument('rootpage', InputArgument::REQUIRED, 'The root page id')
->addArgument('type', InputArgument::OPTIONAL, 'A type. Normally a table name');
->addArgument('rootpage', InputArgument::REQUIRED, 'The root page id')
->addArgument('type', InputArgument::OPTIONAL, 'A type. Normally a table name');
}

protected function execute(InputInterface $input, OutputInterface $output): int
{
/** @var SiteRepository $siteRepository */
$siteRepository = GeneralUtility::makeInstance(SiteRepository::class);

$site = $siteRepository->getSiteByPageId($input->getArgument('rootpage'));
$site = $this->siteRepository->getSiteByPageId((int)$input->getArgument('rootpage'));
if (!$site instanceof Site) {
$output->writeln('Site not found');
return 1;
return Command::FAILURE;
}

$connectionManager = GeneralUtility::makeInstance(ConnectionManager::class);
foreach ($connectionManager->getConnectionsBySite($site) as $connection) {
foreach ($this->connectionManager->getConnectionsBySite($site) as $connection) {
if ($type = $input->getArgument('type')) {
$connection->getWriteService()->deleteByType($type);
} else {
Expand All @@ -41,6 +45,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$connection->getWriteService()->commit();
}

return 0;
return Command::SUCCESS;
}
}
61 changes: 28 additions & 33 deletions Classes/Command/IndexCommand.php
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
<?php

declare(strict_types=1);

namespace Networkteam\SolrCommands\Command;

use ApacheSolrForTypo3\Solr\Domain\Index\IndexService;
use ApacheSolrForTypo3\Solr\Domain\Site\SiteRepository;
use ApacheSolrForTypo3\Solr\Task\IndexQueueWorkerTask;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
Expand All @@ -12,56 +14,49 @@

class IndexCommand extends Command
{
public function configure()
public function __construct(
private readonly SiteRepository $siteRepository,
) {
parent::__construct();
}

protected function configure(): void
{
$this->addArgument('rootpage', InputArgument::OPTIONAL, 'Site root page id', '*');
$this->addArgument('limit', InputArgument::OPTIONAL, 'Limit documents per site')
->setDescription('Index documents of all sites');
$this->setDescription('Index documents of all sites')
->addArgument('rootpage', InputArgument::OPTIONAL, 'Site root page id', '*')
->addArgument('limit', InputArgument::OPTIONAL, 'Limit documents per site');
}

protected function execute(InputInterface $input, OutputInterface $output): int
{
/** @var SiteRepository $siteRepository */
$siteRepository = GeneralUtility::makeInstance(SiteRepository::class);

if (($input->getArgument('rootpage') ?? '*') == '*') {
$sites = $siteRepository->getAvailableSites();
$rootpage = $input->getArgument('rootpage') ?? '*';
if ($rootpage === '*') {
$sites = $this->siteRepository->getAvailableSites();
} else {
$sites = [];
$site = $siteRepository->getSiteByPageId((integer)$input->getArgument('rootpage'));
if ($site) {
$sites[] = $site;
}
$site = $this->siteRepository->getSiteByPageId((int)$rootpage);
$sites = $site ? [$site] : [];
}

if ($input->hasArgument('limit') && (integer)$input->getArgument('limit') > 0) {
$limit = (integer)$input->getArgument('limit');
} else {
$limit = PHP_INT_MAX;
}
$limitArg = (int)($input->getArgument('limit') ?? 0);
$limit = $limitArg > 0 ? $limitArg : PHP_INT_MAX;

$oneTaskFailed = false;
foreach ($sites as $site) {
if ($output->isVerbose()) {
$output->writeln(
sprintf(
'Indexing %s (Root page %d)',
$site->getTypo3SiteObject()->getBase(),
$site->getRootPageId()
)
);
$output->writeln(sprintf(
'Indexing %s (Root page %d)',
$site->getTypo3SiteObject()->getBase(),
$site->getRootPageId(),
));
}

$indexTask = GeneralUtility::makeInstance(IndexQueueWorkerTask::class);
$indexTask->setRootPageId($site->getRootPageId());
$indexTask->setDocumentsToIndexLimit($limit);

if (!$indexTask->execute()) {
$indexService = GeneralUtility::makeInstance(IndexService::class, $site);
if (!$indexService->indexItems($limit)) {
$oneTaskFailed = true;
$output->writeln(sprintf('Failed indexing site %d', $site->getRootPageId()));
}
}

return $oneTaskFailed === true ? Command::FAILURE : Command::SUCCESS;
return $oneTaskFailed ? Command::FAILURE : Command::SUCCESS;
}
}
Loading