From 136ef935fc4e00abeca9022904de5c196831155d Mon Sep 17 00:00:00 2001
From: Tomas Votruba <tomas.vot@gmail.com>
Date: Thu, 17 Sep 2026 22:26:02 +0200
Subject: [PATCH 1/2] Add duplicated-code command for token-based copy-paste
 detection

---
 README.md                                     |  21 ++
 src/Command/DuplicatedCodeCommand.php         |  97 +++++++++
 src/DuplicatedCode/CloneDetector.php          | 196 ++++++++++++++++++
 src/DuplicatedCode/ValueObject/CodeClone.php  |  16 ++
 .../ValueObject/CodeCloneFile.php             |  15 ++
 tests/DuplicatedCode/CloneDetectorTest.php    |  40 ++++
 .../Fixture/first_duplicate.php.inc           |  13 ++
 .../Fixture/second_duplicate.php.inc          |  13 ++
 tests/DuplicatedCode/Fixture/unique.php.inc   |  12 ++
 9 files changed, 423 insertions(+)
 create mode 100644 src/Command/DuplicatedCodeCommand.php
 create mode 100644 src/DuplicatedCode/CloneDetector.php
 create mode 100644 src/DuplicatedCode/ValueObject/CodeClone.php
 create mode 100644 src/DuplicatedCode/ValueObject/CodeCloneFile.php
 create mode 100644 tests/DuplicatedCode/CloneDetectorTest.php
 create mode 100644 tests/DuplicatedCode/Fixture/first_duplicate.php.inc
 create mode 100644 tests/DuplicatedCode/Fixture/second_duplicate.php.inc
 create mode 100644 tests/DuplicatedCode/Fixture/unique.php.inc

diff --git a/README.md b/README.md
index 8a50dee48..0d939a4ce 100644
--- a/README.md
+++ b/README.md
@@ -420,6 +420,27 @@ final class ServiceContainerTest extends AbstractContainerTestCase
 
 <br>
 
+## 12. Detect Duplicated Code
+
+Spot copy-pasted code blocks with a token-based detector, a small clone of phpcpd.
+Add it to CI to fail when a large copy-pasted block is added:
+
+```bash
+vendor/bin/swiss-knife duplicated-code src
+vendor/bin/swiss-knife duplicated-code src rules --min-tokens 150 --min-lines 5
+```
+
+Options:
+
+- `--min-lines` minimum lines of a reported clone (default `5`)
+- `--min-tokens` minimum tokens of a reported clone (default `70`)
+- `--fuzzy` ignore variable names when matching
+- `--skip-file` file paths or masks to skip
+
+Exit code is `1` when clones are found, `0` otherwise.
+
+<br>
+
 That's it!
 
 <br>
diff --git a/src/Command/DuplicatedCodeCommand.php b/src/Command/DuplicatedCodeCommand.php
new file mode 100644
index 000000000..c40a82b7e
--- /dev/null
+++ b/src/Command/DuplicatedCodeCommand.php
@@ -0,0 +1,97 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\Command;
+
+use Entropy\Console\Contract\CommandInterface;
+use Entropy\Console\Enum\ExitCode;
+use Entropy\Console\Output\OutputPrinter;
+use Rector\SwissKnife\DuplicatedCode\CloneDetector;
+use Rector\SwissKnife\Finder\PhpFilesFinder;
+
+final readonly class DuplicatedCodeCommand implements CommandInterface
+{
+    private const int DEFAULT_MIN_LINES = 5;
+
+    private const int DEFAULT_MIN_TOKENS = 70;
+
+    public function __construct(
+        private OutputPrinter $outputPrinter,
+    ) {
+    }
+
+    /**
+     * @param string[] $sources One or more paths to scan
+     * @param string[] $skipFiles File paths to skip
+     * @param int $minLines Minimum lines of a reported clone
+     * @param int $minTokens Minimum tokens of a reported clone
+     * @param bool $fuzzy Ignore variable names when matching
+     *
+     * @return ExitCode::*
+     */
+    public function run(
+        array $sources,
+        array $skipFiles = [],
+        int $minLines = self::DEFAULT_MIN_LINES,
+        int $minTokens = self::DEFAULT_MIN_TOKENS,
+        bool $fuzzy = false
+    ): int {
+        $phpFileInfos = PhpFilesFinder::find($sources, $skipFiles);
+
+        $filePaths = [];
+        foreach ($phpFileInfos as $phpFileInfo) {
+            $filePaths[] = $phpFileInfo->getRealPath();
+        }
+
+        $this->outputPrinter->yellow(sprintf('Scanning %d *.php files for duplicated code', count($filePaths)));
+
+        $cloneDetector = new CloneDetector($minLines, $minTokens, $fuzzy);
+        $clones = $cloneDetector->detect($filePaths);
+
+        if ($clones === []) {
+            $this->outputPrinter->green(sprintf('No duplicates found in %d files', count($filePaths)));
+            return ExitCode::SUCCESS;
+        }
+
+        $duplicatedLines = 0;
+        foreach ($clones as $clone) {
+            $this->outputPrinter->writeln(sprintf(
+                ' * %s:%d-%d (%d lines, %d tokens)',
+                $clone->firstFile->filePath,
+                $clone->firstFile->startLine,
+                $clone->firstFile->endLine,
+                $clone->lines,
+                $clone->tokens
+            ));
+            $this->outputPrinter->writeln(sprintf(
+                '   %s:%d-%d',
+                $clone->secondFile->filePath,
+                $clone->secondFile->startLine,
+                $clone->secondFile->endLine
+            ));
+            $this->outputPrinter->newline();
+
+            $duplicatedLines += $clone->lines;
+        }
+
+        $this->outputPrinter->redBackground(sprintf(
+            'Found %d clones with %d duplicated lines in %d scanned files',
+            count($clones),
+            $duplicatedLines,
+            count($filePaths)
+        ));
+
+        return ExitCode::ERROR;
+    }
+
+    public function getName(): string
+    {
+        return 'duplicated-code';
+    }
+
+    public function getDescription(): string
+    {
+        return 'Finds duplicated, copy-pasted code blocks via token-based detection';
+    }
+}
diff --git a/src/DuplicatedCode/CloneDetector.php b/src/DuplicatedCode/CloneDetector.php
new file mode 100644
index 000000000..49476dbae
--- /dev/null
+++ b/src/DuplicatedCode/CloneDetector.php
@@ -0,0 +1,196 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\DuplicatedCode;
+
+use Rector\SwissKnife\DuplicatedCode\ValueObject\CodeClone;
+use Rector\SwissKnife\DuplicatedCode\ValueObject\CodeCloneFile;
+
+/**
+ * Token-based copy-paste detector using a Rabin-Karp rolling hash over the
+ * normalized token stream, mirroring the classic phpcpd behaviour.
+ * @see \Rector\SwissKnife\Tests\DuplicatedCode\CloneDetectorTest
+ */
+final class CloneDetector
+{
+    /**
+     * Tokens that carry no structural meaning for clone detection.
+     *
+     * @var array<int, true>
+     */
+    private const array IGNORED_TOKENS = [
+        T_INLINE_HTML => true,
+        T_COMMENT => true,
+        T_DOC_COMMENT => true,
+        T_OPEN_TAG => true,
+        T_OPEN_TAG_WITH_ECHO => true,
+        T_CLOSE_TAG => true,
+        T_WHITESPACE => true,
+    ];
+
+    /**
+     * Bytes contributed by each kept token to the signature: 1 type byte + 4 CRC32 bytes.
+     */
+    private const int BYTES_PER_TOKEN = 5;
+
+    /**
+     * First seen location per window hash: hash => [filePath, tokenIndex].
+     *
+     * @var array<string, array{string, int}>
+     */
+    private array $hashes = [];
+
+    /**
+     * Per-file token line numbers, kept for span line lookup.
+     *
+     * @var array<string, int[]>
+     */
+    private array $fileLines = [];
+
+    /**
+     * @var CodeClone[]
+     */
+    private array $clones = [];
+
+    public function __construct(
+        private readonly int $minLines,
+        private readonly int $minTokens,
+        private readonly bool $fuzzy
+    ) {
+    }
+
+    /**
+     * @param string[] $filePaths
+     * @return CodeClone[]
+     */
+    public function detect(array $filePaths): array
+    {
+        foreach ($filePaths as $filePath) {
+            $this->processFile($filePath);
+        }
+
+        return $this->clones;
+    }
+
+    private function processFile(string $filePath): void
+    {
+        $source = file_get_contents($filePath);
+        if ($source === false) {
+            return;
+        }
+
+        [$signature, $lines] = $this->tokenize($source);
+        $this->fileLines[$filePath] = $lines;
+
+        $tokenCount = count($lines);
+        if ($tokenCount < $this->minTokens) {
+            return;
+        }
+
+        $lastWindow = $tokenCount - $this->minTokens;
+        $windowBytes = $this->minTokens * self::BYTES_PER_TOKEN;
+
+        $found = false;
+        $firstToken = 0;
+        $originFile = '';
+        $originToken = 0;
+
+        for ($i = 0; $i <= $lastWindow; ++$i) {
+            $hash = substr(md5(substr($signature, $i * self::BYTES_PER_TOKEN, $windowBytes), true), 0, 8);
+
+            if (isset($this->hashes[$hash])) {
+                if (! $found) {
+                    $found = true;
+                    $firstToken = $i;
+                    [$originFile, $originToken] = $this->hashes[$hash];
+                }
+
+                continue;
+            }
+
+            if ($found) {
+                $this->recordClone($originFile, $originToken, $filePath, $firstToken, $i);
+                $found = false;
+            }
+
+            $this->hashes[$hash] = [$filePath, $i];
+        }
+
+        if ($found) {
+            $this->recordClone($originFile, $originToken, $filePath, $firstToken, $lastWindow + 1);
+        }
+    }
+
+    private function recordClone(
+        string $originFile,
+        int $originToken,
+        string $currentFile,
+        int $currentToken,
+        int $mismatchWindow
+    ): void {
+        $windowCount = $mismatchWindow - $currentToken;
+        $tokenSpan = $windowCount + $this->minTokens - 1;
+
+        $originLines = $this->fileLines[$originFile];
+        $currentLines = $this->fileLines[$currentFile];
+
+        $originStartLine = $originLines[$originToken];
+        $originEndLine = $originLines[min($originToken + $tokenSpan - 1, count($originLines) - 1)];
+
+        $currentStartLine = $currentLines[$currentToken];
+        $currentEndLine = $currentLines[min($currentToken + $tokenSpan - 1, count($currentLines) - 1)];
+
+        $numberOfLines = $originEndLine - $originStartLine + 1;
+        if ($numberOfLines < $this->minLines) {
+            return;
+        }
+
+        $this->clones[] = new CodeClone(
+            new CodeCloneFile($originFile, $originStartLine, $originEndLine),
+            new CodeCloneFile($currentFile, $currentStartLine, $currentEndLine),
+            $numberOfLines,
+            $tokenSpan
+        );
+    }
+
+    /**
+     * Builds the token signature and the parallel line map for a source string.
+     *
+     * @return array{string, int[]}
+     */
+    private function tokenize(string $source): array
+    {
+        $signature = '';
+        $lines = [];
+        $currentLine = 1;
+
+        foreach (token_get_all($source) as $token) {
+            if (is_array($token)) {
+                $tokenId = $token[0];
+                $tokenText = $token[1];
+                $currentLine = $token[2];
+
+                if (isset(self::IGNORED_TOKENS[$tokenId])) {
+                    $currentLine += substr_count($tokenText, "\n");
+                    continue;
+                }
+
+                if ($this->fuzzy && $tokenId === T_VARIABLE) {
+                    $tokenText = '$';
+                }
+
+                $signature .= chr($tokenId & 255) . pack('N*', crc32($tokenText));
+                $lines[] = $currentLine;
+                $currentLine += substr_count($tokenText, "\n");
+
+                continue;
+            }
+
+            $signature .= chr(0) . pack('N*', crc32($token));
+            $lines[] = $currentLine;
+        }
+
+        return [$signature, $lines];
+    }
+}
diff --git a/src/DuplicatedCode/ValueObject/CodeClone.php b/src/DuplicatedCode/ValueObject/CodeClone.php
new file mode 100644
index 000000000..c4ad52a6a
--- /dev/null
+++ b/src/DuplicatedCode/ValueObject/CodeClone.php
@@ -0,0 +1,16 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\DuplicatedCode\ValueObject;
+
+final readonly class CodeClone
+{
+    public function __construct(
+        public CodeCloneFile $firstFile,
+        public CodeCloneFile $secondFile,
+        public int $lines,
+        public int $tokens
+    ) {
+    }
+}
diff --git a/src/DuplicatedCode/ValueObject/CodeCloneFile.php b/src/DuplicatedCode/ValueObject/CodeCloneFile.php
new file mode 100644
index 000000000..57e238dd5
--- /dev/null
+++ b/src/DuplicatedCode/ValueObject/CodeCloneFile.php
@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\DuplicatedCode\ValueObject;
+
+final readonly class CodeCloneFile
+{
+    public function __construct(
+        public string $filePath,
+        public int $startLine,
+        public int $endLine
+    ) {
+    }
+}
diff --git a/tests/DuplicatedCode/CloneDetectorTest.php b/tests/DuplicatedCode/CloneDetectorTest.php
new file mode 100644
index 000000000..07db690c8
--- /dev/null
+++ b/tests/DuplicatedCode/CloneDetectorTest.php
@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\Tests\DuplicatedCode;
+
+use PHPUnit\Framework\TestCase;
+use Rector\SwissKnife\DuplicatedCode\CloneDetector;
+
+final class CloneDetectorTest extends TestCase
+{
+    public function testDetectsDuplicateAcrossFiles(): void
+    {
+        $cloneDetector = new CloneDetector(3, 25, false);
+
+        $clones = $cloneDetector->detect([
+            __DIR__ . '/Fixture/first_duplicate.php.inc',
+            __DIR__ . '/Fixture/second_duplicate.php.inc',
+        ]);
+
+        $this->assertCount(1, $clones);
+
+        $codeClone = $clones[0];
+        $this->assertStringEndsWith('first_duplicate.php.inc', $codeClone->firstFile->filePath);
+        $this->assertStringEndsWith('second_duplicate.php.inc', $codeClone->secondFile->filePath);
+        $this->assertGreaterThanOrEqual(3, $codeClone->lines);
+    }
+
+    public function testDoesNotReportUniqueCode(): void
+    {
+        $cloneDetector = new CloneDetector(3, 25, false);
+
+        $clones = $cloneDetector->detect([
+            __DIR__ . '/Fixture/first_duplicate.php.inc',
+            __DIR__ . '/Fixture/unique.php.inc',
+        ]);
+
+        $this->assertSame([], $clones);
+    }
+}
diff --git a/tests/DuplicatedCode/Fixture/first_duplicate.php.inc b/tests/DuplicatedCode/Fixture/first_duplicate.php.inc
new file mode 100644
index 000000000..dded1cbdc
--- /dev/null
+++ b/tests/DuplicatedCode/Fixture/first_duplicate.php.inc
@@ -0,0 +1,13 @@
+<?php
+
+function firstDuplicate(): int
+{
+    $total = 0;
+    $items = [1, 2, 3, 4, 5];
+
+    foreach ($items as $item) {
+        $total += $item * 2;
+    }
+
+    return $total;
+}
diff --git a/tests/DuplicatedCode/Fixture/second_duplicate.php.inc b/tests/DuplicatedCode/Fixture/second_duplicate.php.inc
new file mode 100644
index 000000000..dded1cbdc
--- /dev/null
+++ b/tests/DuplicatedCode/Fixture/second_duplicate.php.inc
@@ -0,0 +1,13 @@
+<?php
+
+function firstDuplicate(): int
+{
+    $total = 0;
+    $items = [1, 2, 3, 4, 5];
+
+    foreach ($items as $item) {
+        $total += $item * 2;
+    }
+
+    return $total;
+}
diff --git a/tests/DuplicatedCode/Fixture/unique.php.inc b/tests/DuplicatedCode/Fixture/unique.php.inc
new file mode 100644
index 000000000..be01c634f
--- /dev/null
+++ b/tests/DuplicatedCode/Fixture/unique.php.inc
@@ -0,0 +1,12 @@
+<?php
+
+function uniqueLogic(string $name): string
+{
+    $greeting = 'Hello ' . $name;
+
+    if ($name === '') {
+        return 'Hello stranger';
+    }
+
+    return strtoupper($greeting);
+}

From ac28b9d31af1297607ae8d88d924d31806fb14a1 Mon Sep 17 00:00:00 2001
From: Tomas Votruba <tomas.vot@gmail.com>
Date: Thu, 17 Sep 2026 22:47:34 +0200
Subject: [PATCH 2/2] Declare ext-tokenizer, drop removed symplify ECS prepared
 set

---
 composer.json | 1 +
 ecs.php       | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/composer.json b/composer.json
index fd0c4fc94..98ca6801b 100644
--- a/composer.json
+++ b/composer.json
@@ -7,6 +7,7 @@
     ],
     "require": {
         "php": ">=8.4",
+        "ext-tokenizer": "*",
         "entropy/entropy": "^0.4.6",
         "nette/robot-loader": "^4.1",
         "nette/utils": "^4.1",
diff --git a/ecs.php b/ecs.php
index 5634c5b90..55d17833a 100644
--- a/ecs.php
+++ b/ecs.php
@@ -9,6 +9,6 @@
         // invalid syntax test fixture
         __DIR__ . '/tests/PhpParser/Finder/ClassConstantFetchFinder/Fixture/Error/ParseError.php',
     ])
-    ->withPreparedSets(psr12: true, common: true, symplify: true)
+    ->withPreparedSets(psr12: true, common: true)
     ->withPaths([__DIR__ . '/src', __DIR__ . '/tests'])
     ->withRootFiles();