diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e3701e3d7..22f210df3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -21,6 +21,12 @@ jobs: run: node --test --test-concurrency=1 storage.test.js sqliteS3.concurrency.test.js sqliteVercelBlob.etag.test.js sqliteVercelBlob.auth.test.js sqliteVercelBlob.concurrency.test.js working-directory: ./test + # The WordPress update rules. These decide which files an update + # overwrites and deletes in a fork, so they run before anything slow. + - name: WordPress updater unit tests + run: node --test wpUpdate.plan.test.js wpUpdate.apply.test.js wpUpdate.plugins.test.js wpUpdate.themes.test.js wpUpdate.github.test.js + working-directory: ./test + - run: ./run-db-test.sh working-directory: ./test diff --git a/.github/workflows/update-wp.yml b/.github/workflows/update-wp.yml index 3ed995625..4ed933bb7 100644 --- a/.github/workflows/update-wp.yml +++ b/.github/workflows/update-wp.yml @@ -1,25 +1,101 @@ +# Opens a pull request when a new WordPress release comes out. +# +# This runs in every copy of the repository, not just upstream: a site deployed +# from here is a repository nobody logs into, and DISALLOW_FILE_MODS means +# wp-admin will never offer the update either. The pull request is how a site +# owner finds out and how they review the change before it deploys. +# +# Two things have to be switched on by hand in a copy, and neither can be set +# from this file -- see "Keeping WordPress updated" in the readme: +# - Settings > Actions > General > "Allow GitHub Actions to create and +# approve pull requests" +# - a scheduled workflow is disabled automatically after 60 days without a +# push, which is the normal state of a site repository name: Update WordPress on: schedule: - - cron: "0 0 * * *" + # 21:10 UTC. Core release parties start at 15:00 or 17:00 UTC depending on + # the cycle and the package ships an hour or two in, so this picks up a + # release the same day rather than the next one. Cron here is always UTC, so + # the time does not shift with daylight saving. The odd minute is deliberate: + # GitHub queues everything scheduled on the hour together and drops runs when + # that queue is full. + - cron: "10 21 * * *" + # A release that slips past the evening run, or an evening run GitHub drops, + # is picked up twelve hours later instead of twenty-four. + - cron: "10 9 * * *" workflow_dispatch: + +# Repositories created after February 2023 give GITHUB_TOKEN read-only access +# by default. The deploy buttons create a brand new repository, so this applies +# to every copy, and without it the branch push fails in all of them. +permissions: + contents: write + pull-requests: write + jobs: update: name: Update WordPress runs-on: ubuntu-latest - if: github.repository_owner == 'mitchmac' steps: - name: Checkout - uses: actions/checkout@v3 - - run: ./upgrade-wp.sh - working-directory: ./util + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Update WordPress + id: update + run: node util/wp-update --report "${{ runner.temp }}/wp-update.md" + + # The report explains what was left untouched and why, so it is the pull + # request body rather than something only visible in the run log. - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 + if: steps.update.outputs.updated == 'true' + uses: peter-evans/create-pull-request@v7 with: - commit-message: Automated WordPress version update - title: WordPress version update - body: 'This is an automated update of the bundled WordPress files' - committer: "Mitch MacKenzie " - assignees: mitchmac + commit-message: "WordPress ${{ steps.update.outputs.to }}" + title: "Update WordPress to ${{ steps.update.outputs.to }}" + body-path: ${{ runner.temp }}/wp-update.md delete-branch: true branch: 'wordpress-version-update' + + # A separate job on its own branch, so a plugin update is never mixed into a + # core one. Either can be merged, closed or left sitting without affecting + # the other, and a plugin that breaks a site is one revert rather than two. + plugins: + name: Update plugins + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Update plugins + id: update + run: node util/wp-update --plugins --report "${{ runner.temp }}/plugin-update.md" + + - name: Create Pull Request + if: steps.update.outputs.updated == 'true' + uses: peter-evans/create-pull-request@v7 + with: + commit-message: "Update bundled plugins" + title: "Update ${{ steps.update.outputs.plugins }} bundled plugin(s)" + body-path: ${{ runner.temp }}/plugin-update.md + delete-branch: true + branch: 'plugin-version-update' + + # Themes can only be reported on: wordpress.org publishes no checksums + # for them, so an untouched theme can't be told from an edited one. With + # nothing to commit there is no pull request to carry the findings, so + # they go to the run summary. Never fails the job. + - name: Report on themes + if: always() + continue-on-error: true + run: node util/wp-update --themes diff --git a/readme.md b/readme.md index d992e5ecb..6f053bc8f 100644 --- a/readme.md +++ b/readme.md @@ -144,6 +144,32 @@ The most explicitly configured option wins, so adding a Blob store for media won - WordPress and its files are in the ```/wp``` directory. You can add plugins or themes there in their respective directories in ```wp-content``` then commit the files to your repository so it will re-deploy. - Plugins like [Cache-Control](https://wordpress.org/plugins/cache-control/) can enable CDN caching with the s-maxage directive and make your site super fast. Refer to [Vercel Edge Caching](https://vercel.com/docs/concepts/edge-network/caching) or [Netlfiy Cache Headers](https://docs.netlify.com/edge-functions/optional-configuration/#supported-headers) +## Keeping WordPress updated +WordPress lives in your repository, so updates arrive as a pull request instead of through wp-admin. The **Update WordPress** action checks daily for a new release and opens a pull request against your default branch; merging it re-deploys your site. + +Two settings need turning on once in your own copy of the repository: + +1. **Settings → Actions → General → Workflow permissions**, tick *Allow GitHub Actions to create and approve pull requests*. GitHub leaves this off by default and it cannot be enabled from a workflow file. Without it the branch is still pushed, so you can open the pull request yourself. +2. **Actions → Update WordPress → Enable workflow**, if GitHub has disabled it. Scheduled workflows are switched off automatically after 60 days without a push — the normal state of a site repository. You can also run the update at any time with **Run workflow**. + +The update only replaces files that WordPress itself ships, and it checks each one against the checksums wordpress.org publishes before touching it. Your themes, plugins, uploads and `wp-config.php` are never candidates, and neither is a bundled file you have edited or deleted. + +The pull request body lists anything the update skipped and why, along with any core file that differs from what WordPress ships — so an edit you made to WordPress itself shows up before a later release collides with it. + +Bundled plugins are updated the same way, in a **separate** pull request, so a plugin update never rides along with a core one and either can be reverted on its own. A plugin is only replaced when wordpress.org can prove file by file that what's on disk is exactly the release it claims to be — so your own plugins, anything premium, and anything bundled from outside wordpress.org are left alone and listed in the pull request instead. If even one file of a plugin has been edited, the whole plugin is skipped rather than left running a mix of two releases. + +One exception: **SQLite Database Integration** is bundled from its GitHub repository rather than wordpress.org, and follows that repository's default branch. wordpress.org carries an older release of it, so there is nothing to check it against and the pull request diff is the review. Because the copy mirrors the branch, files you add inside that plugin's directory are removed by an update — keep your own code in its own plugin. + +Themes are only ever **reported on**, never updated. wordpress.org publishes no checksums for themes, so there is no way to tell a theme you have edited from an untouched one, and overwriting it would risk your work. Themes bundled with WordPress are excluded from the report because the core update already covers them. Anything else with a newer release is listed in the workflow run summary, and updating it is a manual step. + +To check any of this without changing anything: + +```bash +node util/wp-update --dry-run +node util/wp-update --plugins --dry-run +node util/wp-update --themes +``` + ## Customizing ServerlessWP - `netlify.toml` or `vercel.json` are where we configure ```/api/index.js``` to handle all requests - [mitchmac/serverlesswp-node](https://github.com/mitchmac/serverlesswp-node) is used to run PHP and handle the request diff --git a/test/wpUpdate.apply.test.js b/test/wpUpdate.apply.test.js new file mode 100644 index 000000000..bde5dd767 --- /dev/null +++ b/test/wpUpdate.apply.test.js @@ -0,0 +1,130 @@ +// The file operations in util/wp-update/files.js, against real directories. +// +// The plan tests cover which files get touched; these cover what touching them +// does on disk -- that a write lands, a delete removes only its own file, and +// that a directory holding anything else survives. Core and plugin updates share +// these, so a plugin update is subject to the same rules as a core one. + +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { execFileSync } = require('child_process'); + +const updater = require('../util/wp-update/files.js'); +const core = require('../util/wp-update/core.js'); + +let workDir; +let wpRoot; +let releaseRoot; + +test.beforeEach(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wp-update-test-')); + wpRoot = path.join(workDir, 'wp'); + releaseRoot = path.join(workDir, 'release'); + fs.mkdirSync(wpRoot, { recursive: true }); + fs.mkdirSync(releaseRoot, { recursive: true }); +}); + +test.afterEach(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +function write(root, filePath, contents) { + const file = path.join(root, filePath); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents); +} + +function read(filePath) { + return fs.readFileSync(path.join(wpRoot, filePath), 'utf8'); +} + +function exists(filePath) { + return fs.existsSync(path.join(wpRoot, filePath)); +} + +test('a write creates missing directories and overwrites the old file', () => { + write(wpRoot, 'wp-login.php', 'old'); + write(releaseRoot, 'wp-login.php', 'new'); + write(releaseRoot, 'wp-includes/blocks/new.php', 'added'); + + updater.apply(wpRoot, releaseRoot, { + writes: ['wp-login.php', 'wp-includes/blocks/new.php'], + deletes: [], + }); + + assert.strictEqual(read('wp-login.php'), 'new'); + assert.strictEqual(read('wp-includes/blocks/new.php'), 'added'); +}); + +test('a delete removes the file and prunes the directories it emptied', () => { + write(wpRoot, 'wp-includes/old/deep/gone.php', 'x'); + + updater.apply(wpRoot, releaseRoot, { writes: [], deletes: ['wp-includes/old/deep/gone.php'] }); + + assert.ok(!exists('wp-includes/old/deep/gone.php')); + assert.ok(!exists('wp-includes/old')); + // Pruning stops at the WordPress root even when everything under it went. + assert.ok(fs.existsSync(wpRoot)); +}); + +// The reason nothing here uses rsync --delete: a deleted core file must not +// take a sibling the owner added with it. +test('a delete leaves a file the owner added in the same directory', () => { + write(wpRoot, 'wp-content/plugins/akismet/akismet.php', 'x'); + write(wpRoot, 'wp-content/plugins/akismet/my-notes.txt', 'mine'); + + updater.apply(wpRoot, releaseRoot, { + writes: [], + deletes: ['wp-content/plugins/akismet/akismet.php'], + }); + + assert.ok(!exists('wp-content/plugins/akismet/akismet.php')); + assert.strictEqual(read('wp-content/plugins/akismet/my-notes.txt'), 'mine'); +}); + +test('hashing covers only the paths asked for, and skips what is not there', () => { + write(wpRoot, 'wp-login.php', 'x'); + write(wpRoot, 'wp-config.php', 'secrets'); + + const disk = updater.hashDisk(wpRoot, ['wp-login.php', 'wp-settings.php']); + + // 9dd4e461268c8034f5c8564e155c67a6 is md5('x'). + assert.deepStrictEqual(disk, { 'wp-login.php': '9dd4e461268c8034f5c8564e155c67a6' }); +}); + +// A directory sitting where WordPress ships a file has no md5, and must not +// read as "unmodified core file" and get deleted. +test('a directory where a file is expected is never mistaken for that file', () => { + fs.mkdirSync(path.join(wpRoot, 'wp-login.php')); + + const disk = updater.hashDisk(wpRoot, ['wp-login.php']); + + assert.strictEqual(disk['wp-login.php'], 'not-a-file'); +}); + +test('the installed version comes from wp-includes/version.php', () => { + write(wpRoot, 'wp-includes/version.php', " { + execFileSync('git', ['init', '-q'], { cwd: workDir }); + write(workDir, '.gitignore', 'package-lock.json\n'); + write(wpRoot, 'wp-content/themes/twentytwentyfive/package-lock.json', '{}'); + write(wpRoot, 'wp-login.php', 'x'); + + const ignored = updater.ignoredPaths(wpRoot, [ + 'wp-content/themes/twentytwentyfive/package-lock.json', + 'wp-login.php', + ]); + + assert.deepStrictEqual([...ignored], ['wp-content/themes/twentytwentyfive/package-lock.json']); +}); + +test('nothing is ignored when the copy is not a git repository', () => { + assert.deepStrictEqual([...updater.ignoredPaths(wpRoot, ['wp-login.php'])], []); +}); diff --git a/test/wpUpdate.github.test.js b/test/wpUpdate.github.test.js new file mode 100644 index 000000000..d6bb613c5 --- /dev/null +++ b/test/wpUpdate.github.test.js @@ -0,0 +1,178 @@ +// Following a plugin's GitHub branch, in util/wp-update/github.js and the +// tracked path of util/wp-update/plugins.js. +// +// sqlite-database-integration is bundled from source rather than from +// wordpress.org, so it follows the repository's default branch. There is +// nothing to verify it against: the copy in wp/ mirrors the branch, which means +// this is the one place an update deletes a file it can't account for. The +// tests below pin that behaviour down. + +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { execFileSync } = require('child_process'); + +const github = require('../util/wp-update/github.js'); +const plugins = require('../util/wp-update/plugins.js'); + +let pluginDir; + +test.beforeEach(() => { + pluginDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wp-track-test-')); +}); + +test.afterEach(() => { + fs.rmSync(pluginDir, { recursive: true, force: true }); +}); + +function write(relative, contents) { + const file = path.join(pluginDir, relative); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents); + return file; +} + +// The hash has to be git's, or every file would read as changed and the +// updater would rewrite the whole plugin on every run. +test('the blob hash matches what git computes', () => { + const file = write('hello.txt', 'hello\n'); + + const ours = github.blobSha(fs.readFileSync(file)); + const theirs = execFileSync('git', ['hash-object', file]).toString().trim(); + + assert.strictEqual(ours, theirs); + // Recorded so a change of algorithm can't quietly pass by agreeing with a + // rewritten helper. + assert.strictEqual(ours, 'ce013625030ba8dba906f756967f9e9ca394464a'); +}); + +test('an empty file hashes to git\'s empty blob', () => { + const file = write('empty.txt', ''); + + assert.strictEqual(github.blobSha(fs.readFileSync(file)), 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'); +}); + +test('files are listed relative to the plugin directory', () => { + write('load.php', 'a'); + write('wp-includes/database/load.php', 'b'); + write('integrations/query-monitor/boot.php', 'c'); + + assert.deepStrictEqual(plugins.filesUnder(pluginDir).sort(), [ + 'integrations/query-monitor/boot.php', + 'load.php', + 'wp-includes/database/load.php', + ]); +}); + +test('listing a directory that is not there is empty, not an error', () => { + assert.deepStrictEqual(plugins.filesUnder(path.join(pluginDir, 'nope')), []); +}); + +// Everything below stubs the repository, so the decisions are tested without +// depending on what trunk holds today. +function stubRepo(files) { + const realBranch = github.defaultBranch; + const realTree = github.effectiveTree; + github.defaultBranch = async () => 'trunk'; + github.effectiveTree = async () => new Map(Object.entries(files).map(([p, c]) => [p, github.blobSha(Buffer.from(c))])); + return () => { + github.defaultBranch = realBranch; + github.effectiveTree = realTree; + }; +} + +const TRACKED = { repo: 'WordPress/sqlite-database-integration', path: 'packages/plugin-sqlite-database-integration' }; + +test('a copy matching the branch needs no update', async () => { + write('load.php', 'same'); + const restore = stubRepo({ 'load.php': 'same' }); + + try { + const result = await plugins.inspectTracked({ slug: 'x', dir: pluginDir }, TRACKED); + assert.strictEqual(result.status, 'current'); + } finally { + restore(); + } +}); + +test('a changed file on the branch is written', async () => { + write('load.php', 'old'); + const restore = stubRepo({ 'load.php': 'new' }); + + try { + const result = await plugins.inspectTracked({ slug: 'x', dir: pluginDir }, TRACKED); + assert.strictEqual(result.status, 'track'); + assert.deepStrictEqual(result.plan.writes, ['load.php']); + assert.deepStrictEqual(result.plan.deletes, []); + } finally { + restore(); + } +}); + +test('a file new on the branch is added', async () => { + write('load.php', 'same'); + const restore = stubRepo({ 'load.php': 'same', 'capabilities.php': 'new file' }); + + try { + const result = await plugins.inspectTracked({ slug: 'x', dir: pluginDir }, TRACKED); + assert.deepStrictEqual(result.plan.writes, ['capabilities.php']); + } finally { + restore(); + } +}); + +// The consequence of mirroring a branch, and the one place in this updater +// where a file nobody can account for is removed. Following the branch means +// the directory belongs to the branch. +test('a file the branch does not have is removed, including one added locally', async () => { + write('load.php', 'same'); + write('MY-NOTE.txt', 'my own note'); + write('old/dropped.php', 'gone upstream'); + const restore = stubRepo({ 'load.php': 'same' }); + + try { + const result = await plugins.inspectTracked({ slug: 'x', dir: pluginDir }, TRACKED); + assert.strictEqual(result.status, 'track'); + assert.deepStrictEqual(result.plan.deletes.sort(), ['MY-NOTE.txt', 'old/dropped.php']); + } finally { + restore(); + } +}); + +test('the tracked plugin never goes through wordpress.org', async () => { + // wordpress.org publishes this slug at an older version, so reaching the + // .org path at all would offer a downgrade. + assert.ok(plugins.TRACKED['sqlite-database-integration']); + + write('load.php', 'same'); + const restore = stubRepo({ 'load.php': 'same' }); + + try { + const result = await plugins.inspect({ + slug: 'sqlite-database-integration', + dir: pluginDir, + installed: '3.0.0-rc.7', + }); + assert.strictEqual(result.status, 'current'); + assert.match(result.source, /^WordPress\/sqlite-database-integration@/); + } finally { + restore(); + } +}); + +test('the report names the branch a plugin follows', () => { + const report = plugins.report([ + { + slug: 'sqlite-database-integration', + source: 'WordPress/sqlite-database-integration@trunk', + plan: { writes: ['load.php', 'capabilities.php'], deletes: ['gone.php'] }, + status: 'track', + }, + ]); + + assert.match(report, /follows `WordPress\/sqlite-database-integration@trunk`/); + assert.match(report, /2 file\(s\) changed, 1 removed/); + assert.match(report, /no checksums to check them against/); +}); diff --git a/test/wpUpdate.plan.test.js b/test/wpUpdate.plan.test.js new file mode 100644 index 000000000..7e79e21e3 --- /dev/null +++ b/test/wpUpdate.plan.test.js @@ -0,0 +1,205 @@ +// The rules deciding what a WordPress update touches, in util/wp-update/plan.js. +// +// These run in other people's repositories, against working copies holding +// their themes, plugins and edits. The cases that matter are the ones where +// the plan must decline to act: an occupied path, a locally changed file, a +// bundled plugin the owner deleted on purpose. + +const test = require('node:test'); +const assert = require('node:assert'); + +const planner = require('../util/wp-update/plan.js'); + +// Distinct stand-ins for file contents; only equality matters. +const A = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const B = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; +const MINE = 'cccccccccccccccccccccccccccccccc'; + +function plan(parts) { + return planner.plan({ oldSums: {}, newSums: {}, disk: {}, ...parts }); +} + +test('an untouched core file is updated', () => { + const result = plan({ + oldSums: { 'wp-login.php': A }, + newSums: { 'wp-login.php': B }, + disk: { 'wp-login.php': A }, + }); + + assert.deepStrictEqual(result.writes, ['wp-login.php']); + assert.deepStrictEqual(result.deletes, []); + assert.deepStrictEqual(result.conflicts, []); +}); + +test('a new core file is added', () => { + const result = plan({ + newSums: { 'wp-includes/blocks/new.php': B }, + }); + + assert.deepStrictEqual(result.writes, ['wp-includes/blocks/new.php']); +}); + +test('a file the release drops is removed when it is still verbatim', () => { + const result = plan({ + oldSums: { 'wp-includes/gone.php': A }, + disk: { 'wp-includes/gone.php': A }, + }); + + assert.deepStrictEqual(result.deletes, ['wp-includes/gone.php']); + assert.deepStrictEqual(result.writes, []); +}); + +test('a locally changed core file is reported, not overwritten', () => { + const result = plan({ + oldSums: { 'wp-login.php': A }, + newSums: { 'wp-login.php': B }, + disk: { 'wp-login.php': MINE }, + }); + + assert.deepStrictEqual(result.writes, []); + assert.deepStrictEqual(result.conflicts, [{ path: 'wp-login.php', kind: 'modified' }]); +}); + +test('a locally changed file the release drops is kept, not deleted', () => { + const result = plan({ + oldSums: { 'wp-includes/gone.php': A }, + disk: { 'wp-includes/gone.php': MINE }, + }); + + assert.deepStrictEqual(result.deletes, []); + assert.deepStrictEqual(result.conflicts, [{ path: 'wp-includes/gone.php', kind: 'modified-removed' }]); +}); + +// The case that makes this safe to run in a fork: WordPress starts shipping a +// path the owner already uses. There is no previous checksum proving the file +// was ever ours, so it stays theirs. +test('a path the release adds over an existing file is reported, not overwritten', () => { + const result = plan({ + newSums: { 'wp-content/themes/twentytwentysix/style.css': B }, + disk: { 'wp-content/themes/twentytwentysix/style.css': MINE }, + }); + + assert.deepStrictEqual(result.writes, []); + assert.deepStrictEqual(result.conflicts, [ + { path: 'wp-content/themes/twentytwentysix/style.css', kind: 'occupied' }, + ]); +}); + +test('a file already at the new version is left alone', () => { + const result = plan({ + oldSums: { 'wp-login.php': A }, + newSums: { 'wp-login.php': B }, + disk: { 'wp-login.php': B }, + }); + + assert.deepStrictEqual(result.writes, []); + assert.deepStrictEqual(result.conflicts, []); + assert.strictEqual(result.unchanged, 1); +}); + +// This repository deletes akismet and hello.php, and clones delete bundled +// themes they don't want. An update must not quietly put them back. +test('a deleted bundled file is not restored', () => { + const result = plan({ + oldSums: { 'wp-content/plugins/akismet/akismet.php': A }, + newSums: { 'wp-content/plugins/akismet/akismet.php': B }, + disk: {}, + }); + + assert.deepStrictEqual(result.writes, []); + assert.deepStrictEqual(result.conflicts, []); + assert.deepStrictEqual(result.absent, ['wp-content/plugins/akismet/akismet.php']); +}); + +// A local edit to a core file is reported whether or not this release touches +// it, so it can't sit unnoticed until some later update collides with it. It +// is not a conflict: nothing was skipped on its account. +test('a locally modified core file is reported even when the release leaves it alone', () => { + const result = plan({ + oldSums: { 'wp-load.php': A, 'readme.html': A }, + newSums: { 'wp-load.php': A, 'readme.html': A }, + disk: { 'wp-load.php': MINE, 'readme.html': A }, + }); + + assert.deepStrictEqual(result.writes, []); + assert.deepStrictEqual(result.conflicts, []); + assert.deepStrictEqual(result.localEdits, ['wp-load.php']); + assert.strictEqual(result.unchanged, 1); +}); + +// The counterpart: deleting a bundled file this release doesn't change stays +// silent. Clones drop akismet and whole themes on purpose, and naming those +// files every run would bury everything else. +test('a deleted file this release does not change is not reported', () => { + const result = plan({ + oldSums: { 'wp-content/plugins/akismet/akismet.php': A }, + newSums: { 'wp-content/plugins/akismet/akismet.php': A }, + disk: {}, + }); + + assert.deepStrictEqual(result.absent, []); + assert.deepStrictEqual(result.localEdits, []); + assert.strictEqual(result.unchanged, 1); +}); + +test('an ignored path is left out of the plan entirely', () => { + const result = plan({ + oldSums: { 'wp-content/themes/twentytwentyfive/package-lock.json': A }, + newSums: { 'wp-content/themes/twentytwentyfive/package-lock.json': B }, + disk: { 'wp-content/themes/twentytwentyfive/package-lock.json': MINE }, + ignored: new Set(['wp-content/themes/twentytwentyfive/package-lock.json']), + }); + + assert.deepStrictEqual(result.writes, []); + assert.deepStrictEqual(result.conflicts, []); + assert.strictEqual(result.unchanged, 0); +}); + +// Nothing outside the two checksum lists is a candidate, so a plugin or theme +// the owner added is never named by the plan. +test('files wordpress.org does not list are not considered', () => { + const result = plan({ + oldSums: { 'wp-login.php': A }, + newSums: { 'wp-login.php': B }, + disk: { + 'wp-login.php': A, + 'wp-config.php': MINE, + 'wp-content/plugins/my-plugin/my-plugin.php': MINE, + }, + }); + + assert.deepStrictEqual(result.writes, ['wp-login.php']); + assert.deepStrictEqual(result.deletes, []); + assert.deepStrictEqual(result.conflicts, []); +}); + +test('the report names every conflict and counts the rest', () => { + const result = plan({ + oldSums: { 'wp-login.php': A, 'wp-includes/gone.php': A }, + newSums: { 'wp-login.php': B }, + disk: { 'wp-login.php': MINE, 'wp-includes/gone.php': A }, + }); + + const report = planner.report('7.0.2', '7.1', result); + + assert.match(report, /from 7\.0\.2 to 7\.1/); + assert.match(report, /1 file\(s\) removed/); + assert.match(report, /`wp-login\.php`/); + assert.match(report, /changed locally/); +}); + +// Local edits get their own section: folding them into the untouched count +// would claim the update skipped work it never had. +test('the report separates local edits from what the update skipped', () => { + const result = plan({ + oldSums: { 'wp-login.php': A, 'wp-load.php': A }, + newSums: { 'wp-login.php': B, 'wp-load.php': A }, + disk: { 'wp-login.php': MINE, 'wp-load.php': MINE }, + }); + + const report = planner.report('7.0.2', '7.1', result); + + assert.match(report, /### 1 file\(s\) left untouched/); + assert.match(report, /### 1 locally modified core file\(s\)/); + assert.ok(report.indexOf('left untouched') < report.indexOf('locally modified')); +}); diff --git a/test/wpUpdate.plugins.test.js b/test/wpUpdate.plugins.test.js new file mode 100644 index 000000000..f96e0f795 --- /dev/null +++ b/test/wpUpdate.plugins.test.js @@ -0,0 +1,264 @@ +// Plugin updating in util/wp-update/plugins.js. +// +// The case driving most of this is sqlite-database-integration: it is bundled +// from its GitHub repository at 3.0.0-rc.7, while wordpress.org publishes +// 2.2.23 under the same slug. Code that trusted the slug alone would quietly +// downgrade it. Two independent guards stop that -- the version comparison and +// the absence of published checksums -- and both are tested here. + +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const plugins = require('../util/wp-update/plugins.js'); + +let pluginsRoot; + +test.beforeEach(() => { + pluginsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'wp-plugins-test-')); +}); + +test.afterEach(() => { + fs.rmSync(pluginsRoot, { recursive: true, force: true }); +}); + +function writePlugin(slug, fileName, contents) { + const dir = path.join(pluginsRoot, slug); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, fileName), contents); + return dir; +} + +test('the real bundled plugin is older than the one wordpress.org publishes', () => { + // The exact versions in this repository, and the reason the slug alone + // cannot be trusted. + assert.strictEqual(plugins.compareVersions('2.2.23', '3.0.0-rc.7'), -1); + assert.strictEqual(plugins.compareVersions('3.0.0-rc.7', '2.2.23'), 1); +}); + +test('a release outranks its own release candidate', () => { + assert.strictEqual(plugins.compareVersions('3.0.0', '3.0.0-rc.7'), 1); + assert.strictEqual(plugins.compareVersions('3.0.0-rc.7', '3.0.0-rc.8'), -1); + assert.strictEqual(plugins.compareVersions('1.0.0-beta.1', '1.0.0-rc.1'), -1); + assert.strictEqual(plugins.compareVersions('1.0.0-alpha', '1.0.0-beta'), -1); +}); + +test('version parts compare as numbers, not as text', () => { + assert.strictEqual(plugins.compareVersions('1.0.10', '1.0.9'), 1); + assert.strictEqual(plugins.compareVersions('3.3.1', '3.3.1'), 0); + assert.strictEqual(plugins.compareVersions('1.10', '1.9.9'), 1); +}); + +// An unrecognised suffix must not read as an upgrade, or a plugin bundled from +// a fork could talk the update into overwriting it. +test('an unknown suffix ranks below a plain release', () => { + assert.strictEqual(plugins.compareVersions('1.0.0-mybuild', '1.0.0'), -1); + assert.strictEqual(plugins.compareVersions('1.0.0', '1.0.0-mybuild'), 1); +}); + +// wordpress.org publishes an array of accepted md5s for a re-tagged release -- +// three of tidb-compatibility 1.0.0's four files are like that. Dropping those +// entries made the paths look unknown and the whole plugin read as modified, +// so a matching build has to resolve to the hash on disk. +test('a file matching any accepted build counts as unmodified', () => { + const disk = { 'README.md': 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' }; + const sums = { 'README.md': ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'] }; + + assert.deepStrictEqual(plugins.acceptedHashes(sums, disk), { + 'README.md': 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + }); +}); + +test('a file matching no accepted build stays modified', () => { + const disk = { 'README.md': 'cccccccccccccccccccccccccccccccc' }; + const sums = { 'README.md': ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'] }; + + const resolved = plugins.acceptedHashes(sums, disk); + + assert.strictEqual(resolved['README.md'], 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + assert.notStrictEqual(resolved['README.md'], disk['README.md']); +}); + +test('a single-hash entry is left as it is', () => { + assert.deepStrictEqual( + plugins.acceptedHashes({ LICENSE: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }, {}), + { LICENSE: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }, + ); +}); + +test('the version comes from the plugin header, whatever the file is called', () => { + // WP Offload Media really does keep its header in wordpress-s3.php. + writePlugin('amazon-s3-and-cloudfront', 'wordpress-s3.php', ' { + writePlugin('tidb-compatibility', 'tidb-compatibility.php', ' { + writePlugin('not-a-plugin', 'helper.php', ' { + writePlugin('one', 'one.php', ' p.slug), ['one', 'two']); + assert.deepStrictEqual(found.map((p) => p.installed), ['1.0', '2.0']); +}); + +test('a plugin with a header but no version is left alone', async () => { + writePlugin('mystery', 'mystery.php', ' info[slug] ?? null; + api.pluginChecksums = async (slug, version) => checksums[`${slug}@${version}`] ?? null; + return () => { + api.pluginInfo = realInfo; + api.pluginChecksums = realSums; + }; +} + +test('a plugin wordpress.org does not publish is never touched', async () => { + const dir = writePlugin('my-plugin', 'my-plugin.php', ' { + const dir = writePlugin('bundled-from-source', 'load.php', ' { + const dir = writePlugin('bundled-from-source', 'load.php', ' { + const dir = writePlugin('tidb-compatibility', 'tidb-compatibility.php', ' { + const dir = writePlugin('demo', 'demo.php', 'old'); + const md5Old = '149603e6c03516362a8da23f624db945'; // md5('old') + const restore = stubApi({ + info: { demo: { version: '2.0' } }, + checksums: { + 'demo@1.0': { 'demo.php': md5Old }, + 'demo@2.0': { 'demo.php': 'ffffffffffffffffffffffffffffffff' }, + }, + }); + + try { + const result = await plugins.inspect({ slug: 'demo', dir, installed: '1.0' }); + assert.strictEqual(result.status, 'update'); + assert.deepStrictEqual(result.plan.writes, ['demo.php']); + } finally { + restore(); + } +}); + +// All or nothing: one edited file stops the whole plugin, because a plugin +// running a mix of two releases is worse than one that didn't update. +test('a single edited file stops the whole plugin from updating', async () => { + const dir = writePlugin('demo', 'demo.php', 'old'); + fs.writeFileSync(path.join(dir, 'extra.php'), 'edited by hand'); + const restore = stubApi({ + info: { demo: { version: '2.0' } }, + checksums: { + 'demo@1.0': { 'demo.php': '149603e6c03516362a8da23f624db945', 'extra.php': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }, + 'demo@2.0': { 'demo.php': 'ffffffffffffffffffffffffffffffff', 'extra.php': 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' }, + }, + }); + + try { + const result = await plugins.inspect({ slug: 'demo', dir, installed: '1.0' }); + assert.strictEqual(result.status, 'modified'); + } finally { + restore(); + } +}); + +test('the report names what was updated and what was skipped', () => { + const report = plugins.report([ + { slug: 'tidb-compatibility', installed: '1.0.2', latest: '1.1.0', status: 'update' }, + { slug: 'sqlite-database-integration', installed: '3.0.0-rc.7', status: 'ahead' }, + { slug: 'my-plugin', installed: '1.0', status: 'not-on-org' }, + { slug: 'amazon-s3-and-cloudfront', installed: '3.3.1', status: 'current' }, + ]); + + assert.match(report, /\*\*tidb-compatibility\*\* 1\.0\.2 → 1\.1\.0/); + assert.match(report, /2 plugin\(s\) left untouched/); + assert.match(report, /sqlite-database-integration/); + assert.match(report, /1 plugin\(s\) already up to date/); +}); diff --git a/test/wpUpdate.themes.test.js b/test/wpUpdate.themes.test.js new file mode 100644 index 000000000..b1e67de01 --- /dev/null +++ b/test/wpUpdate.themes.test.js @@ -0,0 +1,165 @@ +// Theme reporting in util/wp-update/themes.js. +// +// This never writes anything -- wordpress.org publishes no theme +// checksums, so an untouched theme can't be told from an edited one. The tests +// that matter are about what it says: a theme bundled with WordPress must not +// be reported as the owner's problem, because the core update already covers it. + +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const themes = require('../util/wp-update/themes.js'); +const api = require('../util/wp-update/api.js'); + +let themesRoot; + +test.beforeEach(() => { + themesRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'wp-themes-test-')); +}); + +test.afterEach(() => { + fs.rmSync(themesRoot, { recursive: true, force: true }); +}); + +function writeTheme(slug, style) { + const dir = path.join(themesRoot, slug); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'style.css'), style); + return dir; +} + +test('the version comes from the style.css header', () => { + writeTheme('twentytwentyfive', '/*\nTheme Name: Twenty Twenty-Five\nVersion: 1.5\n*/\n'); + + assert.strictEqual(themes.readHeader(path.join(themesRoot, 'twentytwentyfive')).version, '1.5'); +}); + +test('a stylesheet without a Theme Name is not a theme', () => { + writeTheme('styles', '/* just some css */\nbody { color: red; }\n'); + + assert.strictEqual(themes.readHeader(path.join(themesRoot, 'styles')), null); +}); + +test('a directory with no style.css is not a theme', () => { + fs.mkdirSync(path.join(themesRoot, 'not-a-theme')); + fs.writeFileSync(path.join(themesRoot, 'not-a-theme', 'readme.txt'), 'stray'); + + assert.strictEqual(themes.readHeader(path.join(themesRoot, 'not-a-theme')), null); +}); + +test('discovery finds themes and skips everything else', () => { + writeTheme('one', '/*\nTheme Name: One\nVersion: 1.0\n*/\n'); + writeTheme('two', '/*\nTheme Name: Two\nVersion: 2.0\n*/\n'); + fs.mkdirSync(path.join(themesRoot, 'empty')); + fs.writeFileSync(path.join(themesRoot, 'index.php'), ' t.slug), ['one', 'two']); +}); + +// Which themes ship with WordPress is read out of the release's own file list, +// so it stays right when WordPress swaps its bundled themes. +test('bundled theme slugs come out of the core checksums', () => { + const bundled = themes.bundledSlugs({ + 'wp-content/themes/twentytwentyfive/style.css': 'a', + 'wp-content/themes/twentytwentyfive/theme.json': 'b', + 'wp-content/themes/twentytwentyfour/style.css': 'c', + 'wp-content/themes/index.php': 'd', + 'wp-login.php': 'e', + }); + + assert.deepStrictEqual([...bundled].sort(), ['twentytwentyfive', 'twentytwentyfour']); +}); + +function stubThemeInfo(byslug) { + const real = api.themeInfo; + api.themeInfo = async (slug) => byslug[slug] ?? null; + return () => { + api.themeInfo = real; + }; +} + +// The case that would otherwise produce a wrong and repeating finding: a theme +// WordPress ships is the core update's job, and .org's standalone listing for it +// says nothing about the copy in wp/. +test('a theme shipped with WordPress is never reported as outdated', async () => { + const restore = stubThemeInfo({ twentytwentyfive: { version: '99.0' } }); + + try { + const result = await themes.inspect( + { slug: 'twentytwentyfive', installed: '1.5' }, + new Set(['twentytwentyfive']), + ); + assert.strictEqual(result.status, 'core'); + assert.ok(!result.latest); + } finally { + restore(); + } +}); + +test('a theme wordpress.org does not publish is the owner to maintain', async () => { + const restore = stubThemeInfo({}); + + try { + const result = await themes.inspect({ slug: 'my-theme', installed: '0.1' }, new Set()); + assert.strictEqual(result.status, 'not-on-org'); + } finally { + restore(); + } +}); + +test('a theme behind its wordpress.org release is reported', async () => { + const restore = stubThemeInfo({ astra: { version: '4.13.8' } }); + + try { + const result = await themes.inspect({ slug: 'astra', installed: '4.0.0' }, new Set()); + assert.strictEqual(result.status, 'outdated'); + assert.strictEqual(result.latest, '4.13.8'); + } finally { + restore(); + } +}); + +test('a theme at or ahead of the published release is current', async () => { + const restore = stubThemeInfo({ astra: { version: '4.13.8' } }); + + try { + assert.strictEqual((await themes.inspect({ slug: 'astra', installed: '4.13.8' }, new Set())).status, 'current'); + assert.strictEqual((await themes.inspect({ slug: 'astra', installed: '5.0' }, new Set())).status, 'current'); + } finally { + restore(); + } +}); + +test('a theme with no version header is not guessed at', async () => { + const restore = stubThemeInfo({ astra: { version: '4.13.8' } }); + + try { + const result = await themes.inspect({ slug: 'astra', installed: undefined }, new Set()); + assert.strictEqual(result.status, 'no-version'); + } finally { + restore(); + } +}); + +test('the report links outdated themes and counts the rest', () => { + const report = themes.report([ + { slug: 'astra', installed: '4.0.0', latest: '4.13.8', status: 'outdated' }, + { slug: 'twentytwentyfive', installed: '1.5', status: 'core' }, + { slug: 'my-theme', installed: '0.1', status: 'not-on-org' }, + ]); + + assert.match(report, /1 theme\(s\) have a newer release/); + assert.match(report, /`astra` 4\.0\.0 → 4\.13\.8/); + assert.match(report, /https:\/\/wordpress\.org\/themes\/astra\//); + assert.match(report, /1 shipped with WordPress/); + assert.match(report, /my-theme/); +}); + +test('a report with nothing outdated says so plainly', () => { + const report = themes.report([{ slug: 'twentytwentyfive', installed: '1.5', status: 'core' }]); + + assert.match(report, /No theme has a newer release/); +}); diff --git a/util/upgrade-wp.sh b/util/upgrade-wp.sh deleted file mode 100755 index 2b910d0ec..000000000 --- a/util/upgrade-wp.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -cd .. -mkdir temp -cd temp -wget https://wordpress.org/latest.zip -unzip latest.zip -cp ../wp/wp-config.php wordpress/ -mkdir wordpress/wp-content/mu-plugins -cp ../wp/wp-content/mu-plugins/serverlesswp.php wordpress/wp-content/mu-plugins/ -rm -rf wordpress/wp-content/plugins/akismet wordpress/wp-content/plugins/hello.php -rm -rf wordpress/wp-content/themes/twentytwentytwo wordpress/wp-content/themes/twentytwentyone -wget https://downloads.wordpress.org/plugin/amazon-s3-and-cloudfront.zip -unzip amazon-s3-and-cloudfront.zip -mv amazon-s3-and-cloudfront wordpress/wp-content/plugins/ -git clone --depth 1 https://github.com/WordPress/sqlite-database-integration.git -cp -rL sqlite-database-integration/packages/plugin-sqlite-database-integration wordpress/wp-content/plugins/sqlite-database-integration -rm -rf sqlite-database-integration -git clone https://github.com/pingcap/wordpress-tidb-plugin.git -wget https://downloads.wordpress.org/plugin/tidb-compatibility.zip -unzip tidb-compatibility -mv tidb-compatibility wordpress/wp-content/plugins/ -rm -rf ../wp -mv wordpress ../wp -cd .. -rm -rf temp \ No newline at end of file diff --git a/util/wp-update/api.js b/util/wp-update/api.js new file mode 100644 index 000000000..961d5b8ad --- /dev/null +++ b/util/wp-update/api.js @@ -0,0 +1,190 @@ +// Everything the core update knows about WordPress comes from +// wordpress.org. The checksums endpoint returns every file a release ships -- +// 3,945 paths for 7.0.2, bundled themes included -- and that list is byte-exact +// against the release zip. So it is the manifest: a path wordpress.org lists is +// WordPress's, and a path it doesn't list belongs to whoever cloned this repo. +// +// wp-config.php is absent from it (only wp-config-sample.php ships), so the +// file most worth protecting is out of scope without special-casing. + +const fs = require('fs'); +const path = require('path'); +const { execFileSync } = require('child_process'); + +const STABLE_CHECK = 'https://api.wordpress.org/core/stable-check/1.0/'; +const CHECKSUMS = 'https://api.wordpress.org/core/checksums/1.0/'; +const RELEASE = 'https://downloads.wordpress.org/release/'; +const PLUGIN_INFO = 'https://api.wordpress.org/plugins/info/1.2/'; +const PLUGIN_CHECKSUMS = 'https://downloads.wordpress.org/plugin-checksums/'; +const PLUGIN_DOWNLOAD = 'https://downloads.wordpress.org/plugin/'; +const THEME_INFO = 'https://api.wordpress.org/themes/info/1.2/'; + +async function getJson(url) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`${url} returned ${response.status}`); + } + return response.json(); +} + +async function download(url, file) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`${url} returned ${response.status}`); + } + fs.writeFileSync(file, Buffer.from(await response.arrayBuffer())); +} + +function unzip(zipPath, into) { + try { + execFileSync('unzip', ['-q', '-o', zipPath, '-d', into], { stdio: 'pipe' }); + } catch (error) { + throw new Error(`could not unzip ${zipPath}: ${error.message}`); + } +} + +// The one version stable-check marks 'latest'. It lists all 800+ releases with +// a status each, so anything but exactly one 'latest' means the shape changed +// and we should stop rather than guess. +exports.latestVersion = async function () { + const versions = await getJson(STABLE_CHECK); + const latest = Object.keys(versions).filter((version) => versions[version] === 'latest'); + + if (latest.length !== 1) { + throw new Error(`stable-check named ${latest.length} latest versions, expected 1`); + } + + return latest[0]; +}; + +// Map of release-relative path -> md5. +exports.checksums = async function (version, locale = 'en_US') { + const body = await getJson(`${CHECKSUMS}?version=${encodeURIComponent(version)}&locale=${encodeURIComponent(locale)}`); + + // A version wordpress.org never published answers 200 with checksums: false. + if (!body || typeof body.checksums !== 'object' || body.checksums === null) { + throw new Error(`no checksums published for WordPress ${version} (${locale})`); + } + + return body.checksums; +}; + +// Downloads and unpacks a release, returning the path to its wordpress/ +// directory. Extraction shells out to unzip, which ubuntu-latest has and Node +// has no equivalent of. +exports.downloadRelease = async function (version, workDir) { + const zipPath = path.join(workDir, `wordpress-${version}.zip`); + const url = `${RELEASE}wordpress-${version}.zip`; + + fs.mkdirSync(workDir, { recursive: true }); + await download(url, zipPath); + unzip(zipPath, workDir); + + const root = path.join(workDir, 'wordpress'); + if (!fs.existsSync(root)) { + throw new Error(`${url} did not contain a wordpress/ directory`); + } + + return root; +}; + +// Tells a wordpress.org plugin from a custom or premium one: the latter answer +// 200 with {"error":"Plugin not found."} rather than a status code. Returns +// null for anything not published there. +exports.pluginInfo = async function (slug) { + const url = `${PLUGIN_INFO}?action=plugin_information&request[slug]=${encodeURIComponent(slug)}`; + const response = await fetch(url); + + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`${url} returned ${response.status}`); + } + + const body = await response.json(); + if (!body || body.error || !body.version) { + return null; + } + + return body; +}; + +// Per-file md5 for one published plugin release, keyed by path relative to the +// plugin directory. Null when wordpress.org has never published that version, +// which is the signal that a bundled plugin came from somewhere else. +// +// A value is a string, or an array of them when a release was re-tagged and +// more than one build is accepted -- three of tidb-compatibility 1.0.2's four +// files are like that. Callers resolve arrays against what is on disk; see +// acceptedHashes in plugins.js. +exports.pluginChecksums = async function (slug, version) { + const url = `${PLUGIN_CHECKSUMS}${encodeURIComponent(slug)}/${encodeURIComponent(version)}.json`; + const response = await fetch(url); + + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`${url} returned ${response.status}`); + } + + const body = await response.json(); + if (!body || typeof body.files !== 'object' || body.files === null) { + return null; + } + + const sums = {}; + for (const [filePath, hashes] of Object.entries(body.files)) { + if (typeof hashes.md5 === 'string' || Array.isArray(hashes.md5)) { + sums[filePath] = hashes.md5; + } + } + + return sums; +}; + +// The published release of a theme, or null if wordpress.org doesn't carry it. +// Unlike the plugin endpoint this answers 404 for an unknown slug rather than +// 200 with an error body. +// +// There is no theme equivalent of pluginChecksums: theme-checksums answers 404 +// for every theme, bundled or not. That absence is why themes are only ever +// reported and never written. +exports.themeInfo = async function (slug) { + const url = `${THEME_INFO}?action=theme_information&request[slug]=${encodeURIComponent(slug)}`; + const response = await fetch(url); + + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`${url} returned ${response.status}`); + } + + const body = await response.json(); + if (!body || body.error || !body.version) { + return null; + } + + return body; +}; + +// Unpacks a plugin release and returns the directory holding its files. +exports.downloadPlugin = async function (slug, version, workDir) { + const into = path.join(workDir, `${slug}-${version}`); + const zipPath = path.join(workDir, `${slug}.${version}.zip`); + const url = `${PLUGIN_DOWNLOAD}${encodeURIComponent(slug)}.${encodeURIComponent(version)}.zip`; + + fs.mkdirSync(into, { recursive: true }); + await download(url, zipPath); + unzip(zipPath, into); + + // The zip wraps everything in a directory named after the plugin. + const root = path.join(into, slug); + if (!fs.existsSync(root)) { + throw new Error(`${url} did not contain a ${slug}/ directory`); + } + + return root; +}; diff --git a/util/wp-update/core.js b/util/wp-update/core.js new file mode 100644 index 000000000..8bfce9507 --- /dev/null +++ b/util/wp-update/core.js @@ -0,0 +1,66 @@ +// Updates the WordPress files themselves. +// +// It only writes paths wordpress.org lists for the version already on disk, so +// themes, plugins, wp-config.php and anything else added to wp/ are never +// candidates. See plan.js for the rules and api.js for where the file list +// comes from. + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +const api = require('./api.js'); +const files = require('./files.js'); +const planner = require('./plan.js'); + +// The version WordPress reports about itself, which is what the checksums for +// the copy on disk have to be requested against. +exports.installedVersion = function (wpRoot) { + const versionFile = path.join(wpRoot, 'wp-includes', 'version.php'); + const match = /\$wp_version\s*=\s*'([^']+)'/.exec(fs.readFileSync(versionFile, 'utf8')); + + if (!match) { + throw new Error(`could not read a version from ${versionFile}`); + } + + return match[1]; +}; + +exports.run = async function (options) { + const from = exports.installedVersion(options.root); + const to = options.target || (await api.latestVersion()); + + if (from === to) { + console.log(`WordPress ${from} is already the latest release.`); + return { updated: false, outputs: { from, to } }; + } + + console.log(`Planning update from WordPress ${from} to ${to}...`); + + const [oldSums, newSums] = await Promise.all([api.checksums(from), api.checksums(to)]); + + const paths = [...new Set([...Object.keys(oldSums), ...Object.keys(newSums)])]; + const plan = planner.plan({ + oldSums, + newSums, + disk: files.hashDisk(options.root, paths), + ignored: files.ignoredPaths(options.root, paths), + }); + + const report = planner.report(from, to, plan); + + if (options.dryRun) { + return { updated: false, report, outputs: { from, to, conflicts: plan.conflicts.length } }; + } + + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wp-update-')); + try { + const releaseRoot = await api.downloadRelease(to, workDir); + files.apply(options.root, releaseRoot, plan); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + + console.log(`Updated ${options.root} to WordPress ${to}.`); + return { updated: true, report, outputs: { from, to, conflicts: plan.conflicts.length } }; +}; diff --git a/util/wp-update/files.js b/util/wp-update/files.js new file mode 100644 index 000000000..4ddde4698 --- /dev/null +++ b/util/wp-update/files.js @@ -0,0 +1,95 @@ +// Reading and writing the working copy, shared by the core and plugin updates. +// +// Nothing here decides what should happen -- plan.js does that -- so every +// function takes an explicit list of paths and never walks a directory looking +// for work. That is what keeps a plugin or theme the owner added invisible to +// an update. + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { execFileSync } = require('child_process'); + +// Stands in for a path occupied by something that isn't a plain file. It can +// never equal a real md5, so plan.js treats it as content it doesn't own. +const NOT_A_FILE = 'not-a-file'; + +exports.NOT_A_FILE = NOT_A_FILE; + +function md5(file) { + return crypto.createHash('md5').update(fs.readFileSync(file)).digest('hex'); +} + +exports.md5 = md5; + +// Hashes only the paths given. Everything else under root is never read, let +// alone written. +exports.hashDisk = function (root, paths) { + const disk = {}; + + for (const filePath of paths) { + const file = path.join(root, filePath); + let stat; + try { + stat = fs.lstatSync(file); + } catch { + continue; + } + disk[filePath] = stat.isFile() && !stat.isSymbolicLink() ? md5(file) : NOT_A_FILE; + } + + return disk; +}; + +// Paths git won't carry into a pull request. Writing them would produce +// invisible changes and a report that repeats on every run. +exports.ignoredPaths = function (root, paths) { + if (!paths.length) { + return new Set(); + } + + let stdout; + try { + stdout = execFileSync('git', ['check-ignore', '-z', '--stdin'], { + cwd: root, + input: paths.join('\0'), + maxBuffer: 64 * 1024 * 1024, + }).toString(); + } catch (error) { + // Exit 1 is "nothing was ignored", the common case. Anything else -- + // no git, not a repository -- means we can't tell. The update then + // treats every path as tracked, which is the behaviour without a + // .gitignore at all; the checksum rules still guard each file. + if (error.status !== 1) { + console.warn(`git check-ignore did not run (${error.message.trim()}); assuming nothing is ignored.`); + } + return new Set(); + } + + return new Set(stdout.split('\0').filter(Boolean)); +}; + +// Directories are only removed once the update has emptied them, so a +// directory still holding anything -- including a file the owner added -- +// stays. +function removeEmptyParents(root, filePath) { + let dir = path.dirname(path.join(root, filePath)); + + while (dir.startsWith(root + path.sep) && fs.readdirSync(dir).length === 0) { + fs.rmdirSync(dir); + dir = path.dirname(dir); + } +} + +exports.apply = function (root, sourceRoot, plan) { + for (const filePath of plan.writes) { + const destination = path.join(root, filePath); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.copyFileSync(path.join(sourceRoot, filePath), destination); + } + + for (const filePath of plan.deletes) { + fs.rmSync(path.join(root, filePath)); + removeEmptyParents(root, filePath); + } +}; diff --git a/util/wp-update/github.js b/util/wp-update/github.js new file mode 100644 index 000000000..2947f051f --- /dev/null +++ b/util/wp-update/github.js @@ -0,0 +1,129 @@ +// Reading a plugin straight out of a GitHub repository. +// +// Some plugins are bundled from GitHub rather than wordpress.org, and follow +// the repository's default branch. There is nothing to verify a copy against -- +// no published checksums, no releases to compare -- so the copy in wp/ mirrors +// the branch and the pull request diff is the review. +// +// A plugin inside a monorepo can be assembled out of more than one directory: +// sqlite-database-integration keeps wp-includes/database as a symlink to +// ../../mysql-on-sqlite/src/, so 20 files in the plugin directory become the 46 +// a site actually runs. Git records that symlink as a blob holding the target +// path, so it is followed here rather than written out as a file full of text. + +const fs = require('fs'); +const path = require('path'); +const posix = path.posix; +const crypto = require('crypto'); +const { execFileSync } = require('child_process'); + +const API = 'https://api.github.com'; +const SYMLINK_MODE = '120000'; + +function headers() { + const value = { accept: 'application/vnd.github+json' }; + // Actions provides a token; using it raises the rate limit and costs + // nothing. Unauthenticated works fine for the handful of calls made here. + if (process.env.GITHUB_TOKEN) { + value.authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + } + return value; +} + +async function getJson(url) { + const response = await fetch(url, { headers: headers() }); + if (!response.ok) { + throw new Error(`${url} returned ${response.status}`); + } + return response.json(); +} + +// How git names a file's contents, so a local file can be compared to a tree +// entry without downloading anything. +exports.blobSha = function (buffer) { + const hash = crypto.createHash('sha1'); + hash.update(`blob ${buffer.length}\0`); + hash.update(buffer); + return hash.digest('hex'); +}; + +exports.defaultBranch = async function (repo) { + const info = await getJson(`${API}/repos/${repo}`); + if (!info.default_branch) { + throw new Error(`${repo} reported no default branch`); + } + return info.default_branch; +}; + +// Every file the plugin is made of, keyed by its path inside the plugin and +// valued by git's hash of its contents. Symlinked directories are spliced in +// from wherever they point, so the result is what the plugin looks like once +// unpacked rather than what its own directory holds. +exports.effectiveTree = async function (repo, ref, subdir) { + const listing = await getJson(`${API}/repos/${repo}/git/trees/${ref}?recursive=1`); + + if (listing.truncated) { + throw new Error(`the file listing for ${repo}@${ref} was truncated`); + } + + const files = new Map(); + + const expand = async (prefix, relativeTo) => { + for (const entry of listing.tree) { + if (entry.type !== 'blob' || !entry.path.startsWith(prefix + '/')) { + continue; + } + + const within = posix.join(relativeTo, entry.path.slice(prefix.length + 1)); + + if (entry.mode !== SYMLINK_MODE) { + files.set(within, entry.sha); + continue; + } + + // The blob holds the target path, relative to the link's own + // directory. + const blob = await getJson(entry.url); + const target = Buffer.from(blob.content, 'base64').toString().replace(/\/$/, ''); + const resolved = posix.normalize(posix.join(posix.dirname(entry.path), target)); + await expand(resolved, within); + } + }; + + await expand(subdir, ''); + + if (!files.size) { + throw new Error(`${repo}@${ref} has no files under ${subdir}`); + } + + return files; +}; + +// Checks the repository out and returns the plugin directory with its symlinks +// turned into real files, which is what has to land in wp/. +// +// This clones rather than downloading a tarball. GitHub builds tarballs with +// git archive, which honours export-ignore, and sqlite-database-integration's +// .gitattributes carries "/packages export-ignore" -- the whole plugin. Its +// tarball is 15KB of readme. A clone ignores export-ignore and gets the files. +exports.materialize = async function (repo, ref, subdir, workDir) { + const checkout = path.join(workDir, 'repo'); + const materialized = path.join(workDir, 'plugin'); + + execFileSync('git', [ + 'clone', '--depth', '1', '--branch', ref, '--quiet', + `https://github.com/${repo}.git`, checkout, + ], { stdio: 'pipe' }); + + const source = path.join(checkout, subdir); + if (!fs.existsSync(source)) { + throw new Error(`${repo}@${ref} has no ${subdir} directory`); + } + + // -L follows the symlinks rather than copying them, so the result stands + // on its own once the rest of the repository is gone. + fs.mkdirSync(materialized, { recursive: true }); + execFileSync('cp', ['-rL', source + '/.', materialized], { stdio: 'pipe' }); + + return materialized; +}; diff --git a/util/wp-update/index.js b/util/wp-update/index.js new file mode 100644 index 000000000..56c54c3f5 --- /dev/null +++ b/util/wp-update/index.js @@ -0,0 +1,100 @@ +// Keeps the bundled WordPress up to date. +// +// node util/wp-update update wp/ to the latest release +// node util/wp-update --plugins update bundled wordpress.org plugins +// node util/wp-update --themes report on themes, change nothing +// node util/wp-update --dry-run report what would change, touch nothing +// +// All three are safe to run in a copy of this repository: a file is +// only written when wordpress.org's checksums prove the copy on disk still +// holds what was published. The two run separately and produce separate pull +// requests, so a plugin update never rides along with a core one. +// +// core.js the WordPress files themselves +// plugins.js bundled plugins from wordpress.org +// themes.js themes, which are only ever reported on +// plan.js the rules deciding what gets written or deleted +// files.js reading and writing the working copy +// versions.js comparing header versions +// api.js where every checksum comes from + +const fs = require('fs'); +const path = require('path'); + +const REPO_ROOT = path.resolve(__dirname, '..', '..'); + +function parseArgs(argv) { + const options = { + root: path.join(REPO_ROOT, 'wp'), + mode: 'core', + target: undefined, + dryRun: false, + report: undefined, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--plugins') { + options.mode = 'plugins'; + } else if (arg === '--themes') { + options.mode = 'themes'; + } else if (arg === '--dry-run') { + options.dryRun = true; + } else if (arg === '--root') { + options.root = path.resolve(argv[++i]); + } else if (arg === '--target') { + options.target = argv[++i]; + } else if (arg === '--report') { + options.report = path.resolve(argv[++i]); + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + return options; +} + +function setOutputs(values) { + if (!process.env.GITHUB_OUTPUT) { + return; + } + + const lines = Object.entries(values).map(([key, value]) => `${key}=${value}`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, lines.join('\n') + '\n'); +} + +const MODES = { + core: './core.js', + plugins: './plugins.js', + themes: './themes.js', +}; + +async function main(argv) { + const options = parseArgs(argv); + const result = await require(MODES[options.mode]).run(options); + + if (result.report) { + console.log('\n' + result.report + '\n'); + if (options.report) { + fs.writeFileSync(options.report, result.report + '\n'); + } + // The theme report produces no commit and so no pull request. Writing to + // the job summary is the only way its findings reach anyone in CI. + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, result.report + '\n'); + } + } + + if (options.dryRun) { + console.log('--dry-run: nothing was written.'); + } + + setOutputs({ updated: String(result.updated && !options.dryRun), ...result.outputs }); +} + +if (require.main === module) { + main(process.argv.slice(2)).catch((error) => { + console.error(error.message); + process.exit(1); + }); +} diff --git a/util/wp-update/plan.js b/util/wp-update/plan.js new file mode 100644 index 000000000..d552d0115 --- /dev/null +++ b/util/wp-update/plan.js @@ -0,0 +1,130 @@ +// Decides what an update does to each file, with no I/O so the rules can be +// tested directly. This code deletes files in other people's repositories, so +// the bias everywhere is to leave a file alone and report it. +// +// The plan works on paths, never on directories: nothing here removes a tree, +// so a theme or plugin the owner added inside wp-content survives by never +// being named. A file is only touched when wordpress.org's checksum for the +// version currently on disk proves the copy still holds what WordPress shipped. + +// Ignored paths are dropped before any of this: git will not carry them into a +// pull request, so writing them would produce changes nobody can review and a +// report that repeats itself on every run. +exports.plan = function ({ oldSums, newSums, disk, ignored = new Set() }) { + const result = { + // Copy from the release into the working copy. + writes: [], + // Remove: WordPress dropped the file and the copy still has it verbatim. + deletes: [], + // Left alone and reported, one entry each: { path, kind }. These are + // the parts of the update that did not happen. + conflicts: [], + // Core files that differ from what WordPress ships but that this + // release doesn't touch, so nothing was skipped on their account. + // Reported anyway: a local edit to a core file is worth knowing about + // whether or not this particular update ran into it. + localEdits: [], + // Files this release ships that the copy doesn't have. Reported as a + // count only -- deleting bundled plugins and themes is normal here, and + // naming all 48 of them on every run would bury the real findings. + absent: [], + unchanged: 0, + }; + + const paths = new Set([...Object.keys(oldSums), ...Object.keys(newSums)]); + + for (const filePath of [...paths].sort()) { + if (ignored.has(filePath)) { + continue; + } + + const before = oldSums[filePath]; + const after = newSums[filePath]; + const onDisk = disk[filePath]; + + if (after && !before) { + // A path this release adds. With no previous checksum there's no + // way to prove an existing file was ever ours, so an occupied path + // is always the owner's. + if (onDisk === undefined) { + result.writes.push(filePath); + } else if (onDisk === after) { + result.unchanged++; + } else { + result.conflicts.push({ path: filePath, kind: 'occupied' }); + } + continue; + } + + if (after && before) { + if (onDisk === after) { + result.unchanged++; + } else if (onDisk === undefined) { + // Absent files are only worth counting when this release + // changes them; deleting bundled plugins and themes is normal + // and the rest of them are nobody's business. + before === after ? result.unchanged++ : result.absent.push(filePath); + } else if (onDisk === before) { + result.writes.push(filePath); + } else if (before === after) { + result.localEdits.push(filePath); + } else { + result.conflicts.push({ path: filePath, kind: 'modified' }); + } + continue; + } + + // Dropped by this release. + if (onDisk === undefined) { + result.unchanged++; + } else if (onDisk === before) { + result.deletes.push(filePath); + } else { + result.conflicts.push({ path: filePath, kind: 'modified-removed' }); + } + } + + return result; +}; + +const CONFLICT_TEXT = { + occupied: 'WordPress now ships this path and the copy already has a different file there', + modified: 'changed locally, so the update was not applied', + 'modified-removed': 'changed locally, so it was kept even though WordPress dropped it', +}; + +// The pull request body. Conflicts come first: they are the part of the update +// that did not happen, and the only part needing a decision. Local edits are +// kept in their own section so the count of what was skipped stays honest. +exports.report = function (from, to, plan) { + const lines = [`Updates the bundled WordPress files from ${from} to ${to}.`, '']; + + lines.push(`- ${plan.writes.length} file(s) added or updated`); + lines.push(`- ${plan.deletes.length} file(s) removed`); + if (plan.absent.length) { + lines.push(`- ${plan.absent.length} file(s) this release changed are not in this copy and were left out`); + } + + if (plan.conflicts.length) { + lines.push('', `### ${plan.conflicts.length} file(s) left untouched`, ''); + lines.push('These were not updated, because doing so would overwrite something this'); + lines.push('repository owns. Nothing here is applied automatically.', ''); + + for (const conflict of plan.conflicts) { + lines.push(`- \`${conflict.path}\` — ${CONFLICT_TEXT[conflict.kind]}`); + } + } + + if (plan.localEdits.length) { + lines.push('', `### ${plan.localEdits.length} locally modified core file(s)`, ''); + lines.push(`These differ from what WordPress ${to} ships. This release doesn't change`); + lines.push('them, so the update skipped nothing, but they will not match upstream and'); + lines.push('a future release touching them will report a conflict.', ''); + + for (const filePath of plan.localEdits) { + lines.push(`- \`${filePath}\``); + } + } + + return lines.join('\n'); +}; diff --git a/util/wp-update/plugins.js b/util/wp-update/plugins.js new file mode 100644 index 000000000..78d09ac57 --- /dev/null +++ b/util/wp-update/plugins.js @@ -0,0 +1,315 @@ +// Updates bundled plugins that came from wordpress.org. +// +// The rule is stricter than it is for core, because a half-updated plugin is +// worse than one left alone: a plugin is only touched when wordpress.org can +// prove, file by file, that what's on disk is exactly the release it claims to +// be. Anything else -- a plugin that isn't on .org, a build .org has never +// published, a single edited file -- is reported and skipped whole. +// +// That is what protects sqlite-database-integration, which is bundled from its +// GitHub repository at a version wordpress.org doesn't carry. It needs no +// entry on any exclusion list: .org publishes no checksums for the installed +// build, so nothing here can prove anything about it and it is left alone. + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +const api = require('./api.js'); +const files = require('./files.js'); +const github = require('./github.js'); +const planner = require('./plan.js'); +const versions = require('./versions.js'); + +// WordPress reads plugin headers from the first 8KB of a file, and so does +// this. The main file is rarely named after the plugin -- WP Offload Media +// lives in wordpress-s3.php, SQLite Database Integration in load.php -- so +// every top-level PHP file is checked for the header rather than guessed at. +const HEADER_BYTES = 8192; + +exports.readHeader = function (pluginDir) { + let entries; + try { + entries = fs.readdirSync(pluginDir); + } catch { + return null; + } + + for (const entry of entries.filter((name) => name.endsWith('.php')).sort()) { + const file = path.join(pluginDir, entry); + if (!fs.statSync(file).isFile()) { + continue; + } + + const head = fs.readFileSync(file).subarray(0, HEADER_BYTES).toString('utf8'); + if (!/^[ \t\/*#@]*Plugin Name:/mi.test(head)) { + continue; + } + + return { mainFile: entry, version: versions.headerField(head, 'Version') }; + } + + return null; +}; + +exports.compareVersions = versions.compareVersions; + +// Every directory under wp-content/plugins holding a plugin header. The slug +// is the directory name, which is what wordpress.org keys plugins by. +exports.discover = function (pluginsRoot) { + let entries; + try { + entries = fs.readdirSync(pluginsRoot, { withFileTypes: true }); + } catch { + return []; + } + + const found = []; + for (const entry of entries.filter((e) => e.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) { + const dir = path.join(pluginsRoot, entry.name); + const header = exports.readHeader(dir); + if (header) { + found.push({ slug: entry.name, dir, installed: header.version }); + } + } + + return found; +}; + +// Flattens the multi-hash entries wordpress.org publishes for re-tagged +// releases down to the one hash that matters here. +// +// plan.js compares one checksum per path, so an array has to collapse before +// it gets there. A file matching any accepted build is the official file, so +// resolving to the hash already on disk is what tells the plan it is untouched. +// With no match, the first accepted hash stands in and the file reads as +// locally modified -- which is what it is. +exports.acceptedHashes = function (sums, disk) { + const resolved = {}; + + for (const [filePath, hashes] of Object.entries(sums)) { + if (!Array.isArray(hashes)) { + resolved[filePath] = hashes; + } else { + resolved[filePath] = hashes.includes(disk[filePath]) ? disk[filePath] : hashes[0]; + } + } + + return resolved; +}; + +// Plugins bundled from GitHub instead of wordpress.org. These follow the +// repository's default branch: whatever it holds is what a site runs, and the +// pull request diff is where that gets reviewed. +// +// wordpress.org carries a plugin under this slug too, at an older version, so +// leaving it out of this list would not merely stop updates -- it would offer a +// downgrade. It is here because the plugin is bundled from source, and it has +// to be listed because nothing in the plugin's own headers points at its +// repository. +exports.TRACKED = { + 'sqlite-database-integration': { + repo: 'WordPress/sqlite-database-integration', + path: 'packages/plugin-sqlite-database-integration', + }, +}; + +// Compares the copy on disk against the repository's default branch. Unlike +// the wordpress.org path there is no proof to be had, so anything differing +// from the branch is replaced and anything the branch doesn't have is removed. +exports.inspectTracked = async function (plugin, tracked) { + const branch = await github.defaultBranch(tracked.repo); + const wanted = await github.effectiveTree(tracked.repo, branch, tracked.path); + + const writes = []; + for (const [filePath, sha] of wanted) { + const file = path.join(plugin.dir, filePath); + let onDisk; + try { + onDisk = fs.readFileSync(file); + } catch { + writes.push(filePath); + continue; + } + if (github.blobSha(onDisk) !== sha) { + writes.push(filePath); + } + } + + const deletes = exports.filesUnder(plugin.dir).filter((filePath) => !wanted.has(filePath)); + + if (!writes.length && !deletes.length) { + return { ...plugin, source: `${tracked.repo}@${branch}`, status: 'current' }; + } + + return { + ...plugin, + source: `${tracked.repo}@${branch}`, + tracked, + branch, + plan: { writes, deletes }, + status: 'track', + }; +}; + +exports.filesUnder = function (root, prefix = '') { + let entries; + try { + entries = fs.readdirSync(root, { withFileTypes: true }); + } catch { + return []; + } + + return entries.flatMap((entry) => + entry.isDirectory() + ? exports.filesUnder(path.join(root, entry.name), prefix + entry.name + '/') + : [prefix + entry.name], + ); +}; + +// Decides what happens to one plugin, without touching it. Every path out of +// here that isn't 'update' or 'track' leaves the plugin exactly as it was. +exports.inspect = async function (plugin) { + const tracked = exports.TRACKED[plugin.slug]; + if (tracked) { + return exports.inspectTracked(plugin, tracked); + } + + if (!plugin.installed) { + return { ...plugin, status: 'no-version' }; + } + + const info = await api.pluginInfo(plugin.slug); + if (!info) { + return { ...plugin, status: 'not-on-org' }; + } + + const latest = info.version; + if (exports.compareVersions(latest, plugin.installed) <= 0) { + // Equal is the common case. Newer-than-.org happens when a plugin is + // bundled from somewhere else, and must never be "updated" backwards. + return { ...plugin, latest, status: exports.compareVersions(plugin.installed, latest) > 0 ? 'ahead' : 'current' }; + } + + // Checksums for what's installed. Their absence is the whole safety net + // for plugins bundled from outside wordpress.org: no proof, no update. + const oldSums = await api.pluginChecksums(plugin.slug, plugin.installed); + if (!oldSums) { + return { ...plugin, latest, status: 'unverifiable' }; + } + + const newSums = await api.pluginChecksums(plugin.slug, latest); + if (!newSums) { + return { ...plugin, latest, status: 'unverifiable' }; + } + + const paths = [...new Set([...Object.keys(oldSums), ...Object.keys(newSums)])]; + const disk = files.hashDisk(plugin.dir, paths); + + const plan = planner.plan({ + oldSums: exports.acceptedHashes(oldSums, disk), + newSums: exports.acceptedHashes(newSums, disk), + disk, + ignored: files.ignoredPaths(plugin.dir, paths), + }); + + // All or nothing. Updating the files that happen to be clean would leave a + // plugin running a mix of two releases, which is worse than not updating. + if (plan.conflicts.length || plan.localEdits.length || plan.absent.length) { + return { ...plugin, latest, status: 'modified', plan }; + } + + return { ...plugin, latest, status: 'update', plan }; +}; + +const SKIP_TEXT = { + 'not-on-org': 'not published on wordpress.org, so there is nothing to compare it against', + 'no-version': 'no Version header, so the installed release is unknown', + ahead: 'newer than the version wordpress.org publishes, so it is left alone', + unverifiable: 'wordpress.org publishes no checksums for the installed build', + modified: 'differs from the release wordpress.org published, so it was not replaced', +}; + +exports.report = function (results) { + const updated = results.filter((r) => r.status === 'update'); + const followed = results.filter((r) => r.status === 'track'); + const current = results.filter((r) => r.status === 'current'); + const skipped = results.filter((r) => SKIP_TEXT[r.status]); + + const lines = ['Updates bundled plugins.', '']; + + for (const plugin of updated) { + lines.push(`- **${plugin.slug}** ${plugin.installed} → ${plugin.latest}`); + } + + for (const plugin of followed) { + const changed = plugin.plan.writes.length; + const removed = plugin.plan.deletes.length; + lines.push( + `- **${plugin.slug}** follows \`${plugin.source}\` — ${changed} file(s) changed` + + (removed ? `, ${removed} removed` : ''), + ); + } + + if (!updated.length && !followed.length) { + lines.push('No plugin updates were available.'); + } + + if (followed.length) { + lines.push('', 'Plugins that follow a branch are not published on wordpress.org, so'); + lines.push('there are no checksums to check them against. The diff below is the'); + lines.push('review.'); + } + + if (skipped.length) { + lines.push('', `### ${skipped.length} plugin(s) left untouched`, ''); + lines.push('These are not updated automatically. Nothing here is applied.', ''); + + for (const plugin of skipped) { + lines.push(`- \`${plugin.slug}\` (${plugin.installed || 'unknown'}) — ${SKIP_TEXT[plugin.status]}`); + } + } + + if (current.length) { + lines.push('', `${current.length} plugin(s) already up to date.`); + } + + return lines.join('\n'); +}; + +exports.run = async function (options) { + const pluginsRoot = path.join(options.root, 'wp-content', 'plugins'); + const plugins = exports.discover(pluginsRoot); + + console.log(`Found ${plugins.length} plugin(s) in ${pluginsRoot}.`); + + const results = []; + for (const plugin of plugins) { + const result = await exports.inspect(plugin); + console.log(` ${plugin.slug} ${plugin.installed || '?'} — ${result.status}`); + results.push(result); + } + + const report = exports.report(results); + const changing = results.filter((result) => result.status === 'update' || result.status === 'track'); + + if (!changing.length || options.dryRun) { + return { updated: false, report, outputs: { plugins: changing.length } }; + } + + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wp-plugins-')); + try { + for (const plugin of changing) { + const source = plugin.status === 'track' + ? await github.materialize(plugin.tracked.repo, plugin.branch, plugin.tracked.path, fs.mkdtempSync(path.join(workDir, 'repo-'))) + : await api.downloadPlugin(plugin.slug, plugin.latest, workDir); + + files.apply(plugin.dir, source, plugin.plan); + console.log(`Updated ${plugin.slug} from ${plugin.source || 'wordpress.org'}.`); + } + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + + return { updated: true, report, outputs: { plugins: changing.length } }; +}; diff --git a/util/wp-update/themes.js b/util/wp-update/themes.js new file mode 100644 index 000000000..a662dec9b --- /dev/null +++ b/util/wp-update/themes.js @@ -0,0 +1,154 @@ +// Reports on themes. This file never writes anything. +// +// wordpress.org publishes no theme-checksums endpoint -- the plugin equivalent +// answers 404 for every theme -- so there is no way to prove a theme on disk is +// still the release it claims to be. Without that proof nothing here writes, +// deletes or downloads anything. A theme update is the owner's to make. +// +// Themes bundled with WordPress are a separate matter: their files are in the +// core checksums, so the core update already covers them. Reporting those as +// outdated would be wrong as well as noisy, and which themes those are comes +// from the same place as everything else -- wordpress.org, for the exact +// WordPress version on disk. + +const fs = require('fs'); +const path = require('path'); + +const api = require('./api.js'); +const core = require('./core.js'); +const versions = require('./versions.js'); + +// A theme is a directory with a style.css carrying a Theme Name header. +exports.readHeader = function (themeDir) { + let style; + try { + style = fs.readFileSync(path.join(themeDir, 'style.css'), 'utf8').slice(0, 8192); + } catch { + return null; + } + + if (!versions.headerField(style, 'Theme Name')) { + return null; + } + + return { version: versions.headerField(style, 'Version') }; +}; + +exports.discover = function (themesRoot) { + let entries; + try { + entries = fs.readdirSync(themesRoot, { withFileTypes: true }); + } catch { + return []; + } + + const found = []; + for (const entry of entries.filter((e) => e.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) { + const header = exports.readHeader(path.join(themesRoot, entry.name)); + if (header) { + found.push({ slug: entry.name, installed: header.version }); + } + } + + return found; +}; + +// The theme directories a WordPress release ships, read out of its own file +// list rather than a list of names kept here. +exports.bundledSlugs = function (checksums) { + const slugs = new Set(); + + for (const filePath of Object.keys(checksums)) { + const parts = filePath.split('/'); + if (parts[0] === 'wp-content' && parts[1] === 'themes' && parts.length > 3) { + slugs.add(parts[2]); + } + } + + return slugs; +}; + +exports.inspect = async function (theme, bundled) { + if (bundled.has(theme.slug)) { + return { ...theme, status: 'core' }; + } + + if (!theme.installed) { + return { ...theme, status: 'no-version' }; + } + + const info = await api.themeInfo(theme.slug); + if (!info) { + return { ...theme, status: 'not-on-org' }; + } + + if (versions.compareVersions(info.version, theme.installed) <= 0) { + return { ...theme, latest: info.version, status: 'current' }; + } + + return { ...theme, latest: info.version, status: 'outdated' }; +}; + +exports.report = function (results) { + const outdated = results.filter((r) => r.status === 'outdated'); + const custom = results.filter((r) => r.status === 'not-on-org' || r.status === 'no-version'); + const bundled = results.filter((r) => r.status === 'core'); + const current = results.filter((r) => r.status === 'current'); + + const lines = ['## Themes', '']; + + if (outdated.length) { + lines.push(`**${outdated.length} theme(s) have a newer release on wordpress.org.**`, ''); + lines.push('Themes are never updated automatically: wordpress.org publishes no'); + lines.push('checksums for them, so there is no way to tell an untouched copy from'); + lines.push('one that has been edited. Updating these is a manual step.', ''); + + for (const theme of outdated) { + lines.push(`- \`${theme.slug}\` ${theme.installed} → ${theme.latest} — https://wordpress.org/themes/${theme.slug}/`); + } + } else { + lines.push('No theme has a newer release on wordpress.org.'); + } + + const notes = []; + if (bundled.length) { + notes.push(`${bundled.length} shipped with WordPress and covered by the core update`); + } + if (current.length) { + notes.push(`${current.length} already up to date`); + } + if (custom.length) { + notes.push(`${custom.length} not published on wordpress.org (${custom.map((t) => t.slug).join(', ')})`); + } + + if (notes.length) { + lines.push('', notes.join('; ') + '.'); + } + + return lines.join('\n'); +}; + +exports.run = async function (options) { + const themesRoot = path.join(options.root, 'wp-content', 'themes'); + const themes = exports.discover(themesRoot); + + console.log(`Found ${themes.length} theme(s) in ${themesRoot}.`); + + // Which themes this WordPress ships, for the version actually on disk. + const wpVersion = core.installedVersion(options.root); + const bundled = exports.bundledSlugs(await api.checksums(wpVersion)); + + const results = []; + for (const theme of themes) { + const result = await exports.inspect(theme, bundled); + console.log(` ${theme.slug} ${theme.installed || '?'} — ${result.status}`); + results.push(result); + } + + // Nothing to commit, ever, so this never reports an update. + return { + updated: false, + report: exports.report(results), + outputs: { outdated: results.filter((r) => r.status === 'outdated').length }, + }; +}; diff --git a/util/wp-update/versions.js b/util/wp-update/versions.js new file mode 100644 index 000000000..6361956fb --- /dev/null +++ b/util/wp-update/versions.js @@ -0,0 +1,78 @@ +// Comparing the version strings plugins and themes put in their headers. +// +// These are not semver and can't be assumed to be. What matters most here is +// that nothing ever reads as an upgrade unless it clearly is: a wrong answer +// sends the plugin update looking to overwrite files with an older release. + +// Ranks below a plain release, so 3.0.0-rc.7 sorts under 3.0.0. An unknown +// word ranks lowest: if we can't place it, we must not call it an upgrade. +const TAG_RANK = { dev: -4, alpha: -3, a: -3, beta: -2, b: -2, rc: -1, pl: 1, p: 1 }; + +function chunks(version) { + return String(version) + .toLowerCase() + .replace(/[-_+]/g, '.') + .replace(/([a-z]+)/g, '.$1.') + .split('.') + .filter((chunk) => chunk !== ''); +} + +function isPreRelease(chunk) { + return !/^\d+$/.test(chunk) && (TAG_RANK[chunk] ?? -5) < 0; +} + +// Returns 1 if a is newer than b, -1 if older, 0 if equal. +exports.compareVersions = function (a, b) { + const left = chunks(a); + const right = chunks(b); + + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const x = left[i]; + const y = right[i]; + + // One version ran out of chunks. A trailing pre-release tag makes it + // the older of the two; a trailing number makes it the newer. + if (x === undefined) { + return isPreRelease(y) ? 1 : -1; + } + if (y === undefined) { + return isPreRelease(x) ? -1 : 1; + } + + const xNumeric = /^\d+$/.test(x); + const yNumeric = /^\d+$/.test(y); + + if (xNumeric && yNumeric) { + if (Number(x) !== Number(y)) { + return Number(x) > Number(y) ? 1 : -1; + } + continue; + } + + // A number outranks any tag, so 3.0.0 beats 3.0.0-rc.7. + if (xNumeric !== yNumeric) { + return xNumeric ? 1 : -1; + } + + const xRank = TAG_RANK[x] ?? -5; + const yRank = TAG_RANK[y] ?? -5; + if (xRank !== yRank) { + return xRank > yRank ? 1 : -1; + } + if (x !== y) { + return x > y ? 1 : -1; + } + } + + return 0; +}; + +// Reads one field out of a WordPress file header, which may be laid out as a +// plain comment or a doc block: +// +// Version: 1.0.2 +// * Version: 1.0.2 +exports.headerField = function (text, field) { + const match = new RegExp(`^[ \\t/*#@]*${field}:(.*)$`, 'mi').exec(text); + return match ? match[1].trim() : undefined; +};