Add duplicated-code command for token-based copy-paste detection

README.md

@@ -420,6 +420,27 @@ final class ServiceContainerTest extends AbstractContainerTestCase
 
 <br>
 
+## 12. Detect Duplicated Code
+
+Spot copy-pasted code blocks with a token-based detector, a small clone of phpcpd.
+Add it to CI to fail when a large copy-pasted block is added:
+
+```bash
+vendor/bin/swiss-knife duplicated-code src
+vendor/bin/swiss-knife duplicated-code src rules --min-tokens 150 --min-lines 5
+```
+
+Options:
+
+- `--min-lines` minimum lines of a reported clone (default `5`)
+- `--min-tokens` minimum tokens of a reported clone (default `70`)
+- `--fuzzy` ignore variable names when matching
+- `--skip-file` file paths or masks to skip
+
+Exit code is `1` when clones are found, `0` otherwise.
+
+<br>
+
 That's it!
 
 <br>

composer.json

@@ -7,6 +7,7 @@
     ],
     "require": {
         "php": ">=8.4",
+        "ext-tokenizer": "*",
         "entropy/entropy": "^0.4.6",
         "nette/robot-loader": "^4.1",
         "nette/utils": "^4.1",

ecs.php

@@ -9,6 +9,6 @@
         // invalid syntax test fixture
         __DIR__ . '/tests/PhpParser/Finder/ClassConstantFetchFinder/Fixture/Error/ParseError.php',
     ])
-    ->withPreparedSets(psr12: true, common: true, symplify: true)
+    ->withPreparedSets(psr12: true, common: true)
     ->withPaths([__DIR__ . '/src', __DIR__ . '/tests'])
     ->withRootFiles();

src/Command/DuplicatedCodeCommand.php

@@ -0,0 +1,97 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\Command;
+
+use Entropy\Console\Contract\CommandInterface;
+use Entropy\Console\Enum\ExitCode;
+use Entropy\Console\Output\OutputPrinter;
+use Rector\SwissKnife\DuplicatedCode\CloneDetector;
+use Rector\SwissKnife\Finder\PhpFilesFinder;
+
+final readonly class DuplicatedCodeCommand implements CommandInterface
+{
+    private const int DEFAULT_MIN_LINES = 5;
+
+    private const int DEFAULT_MIN_TOKENS = 70;
+
+    public function __construct(
+        private OutputPrinter $outputPrinter,
+    ) {
+    }
+
+    /**
+     * @param string[] $sources One or more paths to scan
+     * @param string[] $skipFiles File paths to skip
+     * @param int $minLines Minimum lines of a reported clone
+     * @param int $minTokens Minimum tokens of a reported clone
+     * @param bool $fuzzy Ignore variable names when matching
+     *
+     * @return ExitCode::*
+     */
+    public function run(
+        array $sources,
+        array $skipFiles = [],
+        int $minLines = self::DEFAULT_MIN_LINES,
+        int $minTokens = self::DEFAULT_MIN_TOKENS,
+        bool $fuzzy = false
+    ): int {
+        $phpFileInfos = PhpFilesFinder::find($sources, $skipFiles);
+
+        $filePaths = [];
+        foreach ($phpFileInfos as $phpFileInfo) {
+            $filePaths[] = $phpFileInfo->getRealPath();
+        }
+
+        $this->outputPrinter->yellow(sprintf('Scanning %d *.php files for duplicated code', count($filePaths)));
+
+        $cloneDetector = new CloneDetector($minLines, $minTokens, $fuzzy);
+        $clones = $cloneDetector->detect($filePaths);
+
+        if ($clones === []) {
+            $this->outputPrinter->green(sprintf('No duplicates found in %d files', count($filePaths)));
+            return ExitCode::SUCCESS;
+        }
+
+        $duplicatedLines = 0;
+        foreach ($clones as $clone) {
+            $this->outputPrinter->writeln(sprintf(
+                ' * %s:%d-%d (%d lines, %d tokens)',
+                $clone->firstFile->filePath,
+                $clone->firstFile->startLine,
+                $clone->firstFile->endLine,
+                $clone->lines,
+                $clone->tokens
+            ));
+            $this->outputPrinter->writeln(sprintf(
+                '   %s:%d-%d',
+                $clone->secondFile->filePath,
+                $clone->secondFile->startLine,
+                $clone->secondFile->endLine
+            ));
+            $this->outputPrinter->newline();
+
+            $duplicatedLines += $clone->lines;
+        }
+
+        $this->outputPrinter->redBackground(sprintf(
+            'Found %d clones with %d duplicated lines in %d scanned files',
+            count($clones),
+            $duplicatedLines,
+            count($filePaths)
+        ));
+
+        return ExitCode::ERROR;
+    }
+
+    public function getName(): string
+    {
+        return 'duplicated-code';
+    }
+
+    public function getDescription(): string
+    {
+        return 'Finds duplicated, copy-pasted code blocks via token-based detection';
+    }
+}

src/DuplicatedCode/CloneDetector.php

@@ -0,0 +1,196 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\DuplicatedCode;
+
+use Rector\SwissKnife\DuplicatedCode\ValueObject\CodeClone;
+use Rector\SwissKnife\DuplicatedCode\ValueObject\CodeCloneFile;
+
+/**
+ * Token-based copy-paste detector using a Rabin-Karp rolling hash over the
+ * normalized token stream, mirroring the classic phpcpd behaviour.
+ * @see \Rector\SwissKnife\Tests\DuplicatedCode\CloneDetectorTest
+ */
+final class CloneDetector
+{
+    /**
+     * Tokens that carry no structural meaning for clone detection.
+     *
+     * @var array<int, true>
+     */
+    private const array IGNORED_TOKENS = [
+        T_INLINE_HTML => true,
+        T_COMMENT => true,
+        T_DOC_COMMENT => true,
+        T_OPEN_TAG => true,
+        T_OPEN_TAG_WITH_ECHO => true,
+        T_CLOSE_TAG => true,
+        T_WHITESPACE => true,
+    ];
+
+    /**
+     * Bytes contributed by each kept token to the signature: 1 type byte + 4 CRC32 bytes.
+     */
+    private const int BYTES_PER_TOKEN = 5;
+
+    /**
+     * First seen location per window hash: hash => [filePath, tokenIndex].
+     *
+     * @var array<string, array{string, int}>
+     */
+    private array $hashes = [];
+
+    /**
+     * Per-file token line numbers, kept for span line lookup.
+     *
+     * @var array<string, int[]>
+     */
+    private array $fileLines = [];
+
+    /**
+     * @var CodeClone[]
+     */
+    private array $clones = [];
+
+    public function __construct(
+        private readonly int $minLines,
+        private readonly int $minTokens,
+        private readonly bool $fuzzy
+    ) {
+    }
+
+    /**
+     * @param string[] $filePaths
+     * @return CodeClone[]
+     */
+    public function detect(array $filePaths): array
+    {
+        foreach ($filePaths as $filePath) {
+            $this->processFile($filePath);
+        }
+
+        return $this->clones;
+    }
+
+    private function processFile(string $filePath): void
+    {
+        $source = file_get_contents($filePath);
+        if ($source === false) {
+            return;
+        }
+
+        [$signature, $lines] = $this->tokenize($source);
+        $this->fileLines[$filePath] = $lines;
+
+        $tokenCount = count($lines);
+        if ($tokenCount < $this->minTokens) {
+            return;
+        }
+
+        $lastWindow = $tokenCount - $this->minTokens;
+        $windowBytes = $this->minTokens * self::BYTES_PER_TOKEN;
+
+        $found = false;
+        $firstToken = 0;
+        $originFile = '';
+        $originToken = 0;
+
+        for ($i = 0; $i <= $lastWindow; ++$i) {
+            $hash = substr(md5(substr($signature, $i * self::BYTES_PER_TOKEN, $windowBytes), true), 0, 8);
+
+            if (isset($this->hashes[$hash])) {
+                if (! $found) {
+                    $found = true;
+                    $firstToken = $i;
+                    [$originFile, $originToken] = $this->hashes[$hash];
+                }
+
+                continue;
+            }
+
+            if ($found) {
+                $this->recordClone($originFile, $originToken, $filePath, $firstToken, $i);
+                $found = false;
+            }
+
+            $this->hashes[$hash] = [$filePath, $i];
+        }
+
+        if ($found) {
+            $this->recordClone($originFile, $originToken, $filePath, $firstToken, $lastWindow + 1);
+        }
+    }
+
+    private function recordClone(
+        string $originFile,
+        int $originToken,
+        string $currentFile,
+        int $currentToken,
+        int $mismatchWindow
+    ): void {
+        $windowCount = $mismatchWindow - $currentToken;
+        $tokenSpan = $windowCount + $this->minTokens - 1;
+
+        $originLines = $this->fileLines[$originFile];
+        $currentLines = $this->fileLines[$currentFile];
+
+        $originStartLine = $originLines[$originToken];
+        $originEndLine = $originLines[min($originToken + $tokenSpan - 1, count($originLines) - 1)];
+
+        $currentStartLine = $currentLines[$currentToken];
+        $currentEndLine = $currentLines[min($currentToken + $tokenSpan - 1, count($currentLines) - 1)];
+
+        $numberOfLines = $originEndLine - $originStartLine + 1;
+        if ($numberOfLines < $this->minLines) {
+            return;
+        }
+
+        $this->clones[] = new CodeClone(
+            new CodeCloneFile($originFile, $originStartLine, $originEndLine),
+            new CodeCloneFile($currentFile, $currentStartLine, $currentEndLine),
+            $numberOfLines,
+            $tokenSpan
+        );
+    }
+
+    /**
+     * Builds the token signature and the parallel line map for a source string.
+     *
+     * @return array{string, int[]}
+     */
+    private function tokenize(string $source): array
+    {
+        $signature = '';
+        $lines = [];
+        $currentLine = 1;
+
+        foreach (token_get_all($source) as $token) {
+            if (is_array($token)) {
+                $tokenId = $token[0];
+                $tokenText = $token[1];
+                $currentLine = $token[2];
+
+                if (isset(self::IGNORED_TOKENS[$tokenId])) {
+                    $currentLine += substr_count($tokenText, "\n");
+                    continue;
+                }
+
+                if ($this->fuzzy && $tokenId === T_VARIABLE) {
+                    $tokenText = '$';
+                }
+
+                $signature .= chr($tokenId & 255) . pack('N*', crc32($tokenText));
+                $lines[] = $currentLine;
+                $currentLine += substr_count($tokenText, "\n");
+
+                continue;
+            }
+
+            $signature .= chr(0) . pack('N*', crc32($token));
+            $lines[] = $currentLine;
+        }
+
+        return [$signature, $lines];
+    }
+}

src/DuplicatedCode/ValueObject/CodeClone.php

@@ -0,0 +1,16 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\DuplicatedCode\ValueObject;
+
+final readonly class CodeClone
+{
+    public function __construct(
+        public CodeCloneFile $firstFile,
+        public CodeCloneFile $secondFile,
+        public int $lines,
+        public int $tokens
+    ) {
+    }
+}

src/DuplicatedCode/ValueObject/CodeCloneFile.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\SwissKnife\DuplicatedCode\ValueObject;
+
+final readonly class CodeCloneFile
+{
+    public function __construct(
+        public string $filePath,
+        public int $startLine,
+        public int $endLine
+    ) {
+    }
+}