diff --git a/NEWS.md b/NEWS.md index 055d55a44..bbcc253d4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,6 +6,46 @@ width: 128px; border-radius: 128px; " /> +## v2.2.11 + +- Features + - Edit a comic's publish date in the tag editor. Year, month and day are + separate, so a year-only comic stays that way. Editing a date rewrites the + whole date tag: a MetronInfo StoreDate does not survive it, and a + multi-comic edit overwrites parts they disagree on. + - Country, Collection Title and Alternative Issue are editable too, having + only ever been displayed. + - Redesigned the Admin Tagging Status table to be more informative. + +- Fixes + - Bookmarks and read progress survive tag writes. Comics are renamed before + they are written and the database moves with the file, so a rename, a CBR + conversion, or a library scan landing mid-batch can no longer lose them. + PDFs lost them every time before; other formats occasionally. + - Comics are no longer deleted while their files are still on disk. One + replaced in place, by a tool that removes and rewrites it, is re-read + instead of deleted and re-added as a new comic. + - A library that has gone missing, like an unmounted share or volume, no + longer has all of its comics deleted. + - Paths that merely begin the same way are no longer confused for one + another: deleting the folder "Batman" leaves "Batman Beyond" alone, and + libraries at /comics and /comics-kids no longer claim each other's + changes. + - Renaming keeps each archive's own file extension. PDFs and unconverted + CBRs were renamed to .cbz names. + - Two comics that would end up with the same filename no longer overwrite + each other; the second is reported instead. + - Editing a comic's tags twice in a row no longer fails the second time with + a "no such file" error. + - Online tagging only offers Resume when a scan can actually resume, and + reports a failed pause or resume instead of silently doing nothing. + - A second online tagging scan no longer re-queues comics the running scan + already has. + - Deleting a folder refreshes the series and publishers it emptied, which + kept listing comics that were gone. + - The librarian shuts down cleanly with work queued, and tasks queued in the + same instant no longer collide and lose one. + ## v2.2.10 - Fixes diff --git a/bun.lock b/bun.lock index 5526d00f7..6314fd405 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,5 @@ { - "lockfileVersion": 1, + "lockfileVersion": 2, "configVersion": 1, "workspaces": { "": { @@ -11,7 +11,7 @@ "@prettier/plugin-xml": "^3.4.2", "@stylistic/eslint-plugin": "^5.10.0", "@vitest/eslint-plugin": "^1.6.27", - "eslint": "^10.9.0", + "eslint": "^10.9.1", "eslint-config-prettier": "^10.1.8", "eslint-plugin-array-func": "^5.1.1", "eslint-plugin-compat": "^7.0.2", @@ -48,7 +48,7 @@ "remark-preset-lint-markdown-style-guide": "^6.0.1", "remark-preset-lint-recommended": "^7.0.1", "remark-preset-prettier": "^2.0.2", - "svgo": "^4.0.2", + "svgo": "^4.1.0", }, }, }, @@ -172,19 +172,19 @@ "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.67.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.67.0", "@typescript-eslint/types": "^8.67.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-cvE8c7ulYeXN9fYuszhCeCsbzyVEXuhrRCybnBre7TUmqb5nRmBfQAwCj0O3WJFDeyAZt4VYv51vMCC9LHSdYw=="], + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.68.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.68.0", "@typescript-eslint/types": "^8.68.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-5GQtWZCXFcFYux955pvoS02WLc49pXNlvIxocKjS0clvwo3in1RdlzVKyiqQH9vE5AKWFLTaUgeQkOrTS+0Qxw=="], - "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0" } }, "sha512-EgvsleTwS4E+WzzSvem8fAUubLwatMNF1B5hHSLQxcvs7q2dtRhGyujHwLJSYlG41niJ7GP24Aha2+0mb1b2kg=="], + "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.68.0", "", { "dependencies": { "@typescript-eslint/types": "8.68.0", "@typescript-eslint/visitor-keys": "8.68.0" } }, "sha512-T5eXpcaJNg8bhjHJ8Rjp68Vq/QBteYtTKY8TZqVNPaUbuz0f6jI9t6aDkylwvalpAB9XTTFeFOjrjXAZ3YvmVA=="], - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.67.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-vV+LUSv5njUWsknE71fqKTlXUva+R76SaeORd6Zojcunk/6DvKFXONU3BrAs2H49mbygUXt6gbYunzwqNwlhdg=="], + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.68.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-F7zrGQfiJHojPwi8vhxZQC1tWtJzvL74cK/nqri2lk8YUXvYaYwl263xOJ69jDWPUk1hmcdoayFwk9lX09npVw=="], - "@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], + "@typescript-eslint/types": ["@typescript-eslint/types@8.68.0", "", {}, "sha512-9RnpsGJjrAllCMefGVVsImJM24YurhC0Q1h4UbvivtvOqXmR/vEJge2OoE++z9m6hyg8T1Q8t5SNT6tHSbrxcg=="], - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.67.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.67.0", "@typescript-eslint/tsconfig-utils": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-EKQBCE9yNlRJYm7jdTW5AhDacDUmSwQb0FAJAmK2EKYrNXIsa2vxcSZx6PvJ/dEdI6lS+Y9W+EXckLj0iPFGcw=="], + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.68.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.68.0", "@typescript-eslint/tsconfig-utils": "8.68.0", "@typescript-eslint/types": "8.68.0", "@typescript-eslint/visitor-keys": "8.68.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-OKKsD0tYmoNiU5PW2zehO1yO56jYOm1ShYlxon/Z0SJNidAkdVg86eg9ruRuoXf8xfnuWZGbwDsStkoXbZtIIA=="], - "@typescript-eslint/utils": ["@typescript-eslint/utils@8.67.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-U9D1FdwEWBwok3hxxSdhclMb0twvt9QnjIQ0VfQ1AiX2epnpSgv2ubVDsayOFyY8K6FX+AQ7E0FKWVG3iKsj1A=="], + "@typescript-eslint/utils": ["@typescript-eslint/utils@8.68.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.68.0", "@typescript-eslint/types": "8.68.0", "@typescript-eslint/typescript-estree": "8.68.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-PB5gJMMOg0Q5P1tsgWtEAqQacJXq0qEqRHDX/YJ4FaTMLfZPpHB3gjl2EJuiZyPABxmj4ZQYiY9m1bdAJ5y7tQ=="], - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-fkv8dHRDqfGtTHuJeebdrQ7cX6Ad4WAS00rgHh9UGvMycF1mjBfsxry1XsLIFhWZ6Judlh6UdzK+TYlbpCXgnA=="], + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.68.0", "", { "dependencies": { "@typescript-eslint/types": "8.68.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-YR65gGdGvTUAWLldC3xLOvOzamdGzB4A5/N8rehEaHs3Zvoe39BhgY+u0SPch1OvrVTfLcc55wsSgK2NcnTS/A=="], "@unrs/resolver-binding-android-arm-eabi": ["@unrs/resolver-binding-android-arm-eabi@1.12.2", "", { "os": "android", "cpu": "arm" }, "sha512-g5T90pqg1bo/7mytQx6F4iBNC0Wsh9cu+z9veDbFjc7HjpesJFWD7QMS0NGStXM075+7dJPPVvBbpZlnrdpi/w=="], @@ -254,7 +254,7 @@ "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], - "baseline-browser-mapping": ["baseline-browser-mapping@2.11.18", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-1iEmLEYSiE1SeBoAfPo/Mnx3PzfzHUkDK61ASkCpuk3YXugYLH5DYK1SzqV55F8FMI6s0F+/tCP7Polz1QRjxw=="], + "baseline-browser-mapping": ["baseline-browser-mapping@2.11.19", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-Grytf1xOxOEMTGRwx6rLGKkTabd4vMg3VrKdj/7joCmV0qgh4QwMMO6xh34YEXQqirAuUdgQGa5orJQQ+69RBw=="], "binary-extensions": ["binary-extensions@2.3.0", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="], @@ -272,7 +272,7 @@ "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], - "caniuse-lite": ["caniuse-lite@1.0.30001809", "", {}, "sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ=="], + "caniuse-lite": ["caniuse-lite@1.0.30001810", "", {}, "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg=="], "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], @@ -316,11 +316,11 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], - "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + "css-select": ["css-select@6.0.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^7.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "nth-check": "^2.1.1" } }, "sha512-rZZVSLle8v0+EY8QAkDWrKhpgt6SA5OtHsgBnsj6ZaLb5dmDVOWUDtQitd9ydxxvEjhewNudS6eTVU7uOyzvXw=="], "css-tree": ["css-tree@3.2.1", "", { "dependencies": { "mdn-data": "2.27.1", "source-map-js": "^1.2.1" } }, "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA=="], - "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + "css-what": ["css-what@7.0.0", "", {}, "sha512-wD5oz5xibMOPHzy13CyGmogB3phdvcDaB5t0W/Nr5Z2O/agcB8YwOz6e2Lsp10pNDzBoDO9nVa3RGs/2BttpHQ=="], "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="], @@ -352,7 +352,7 @@ "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], - "electron-to-chromium": ["electron-to-chromium@1.5.412", "", {}, "sha512-z4rMe3esBzlzovKHj4gxJnsCGZRK5l4baUvm+gCGJBPE+gsyUMKsuU9tnEUtI1dOebXz1ytAPGjvXhmQ7rIPwA=="], + "electron-to-chromium": ["electron-to-chromium@1.5.415", "", {}, "sha512-958V+Kbhtgz+SxXeEVKBjrlKRBIDAYvUJfwhjxMZ5S6ut9jAl7l9ZKBkBrvjyjZE36PabLUo2L8kEeV5O4vgJg=="], "emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], @@ -370,7 +370,7 @@ "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], - "eslint": ["eslint@10.9.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.5", "@eslint/config-helpers": "^0.7.0", "@eslint/core": "^1.2.1", "@eslint/plugin-kit": "^0.7.2", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "minimatch": "^10.2.5", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "peerDependencies": { "jiti": "*" }, "optionalPeers": ["jiti"], "bin": { "eslint": "bin/eslint.js" } }, "sha512-5KeEOJZBfEVA47boFiBsf+6MmmJpffM7qEBg4pLla2e4nlKgdKlqCW0oSLOGsT8Wl5uCGJptLV1bkaiShj90Gw=="], + "eslint": ["eslint@10.9.1", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.5", "@eslint/config-helpers": "^0.7.0", "@eslint/core": "^1.2.1", "@eslint/plugin-kit": "^0.7.2", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "minimatch": "^10.2.5", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "peerDependencies": { "jiti": "*" }, "optionalPeers": ["jiti"], "bin": { "eslint": "bin/eslint.js" } }, "sha512-9VaAkDURekixUQJy0oJYl2DcN6oKMfxay7XzaGYAWQwsb6qfKf+x76R2k1L8kb1boc+FyCAaTA9GmiKaaiaF+A=="], "eslint-config-prettier": ["eslint-config-prettier@10.1.8", "", { "peerDependencies": { "eslint": ">=7.0.0" }, "bin": { "eslint-config-prettier": "bin/cli.js" } }, "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w=="], @@ -558,7 +558,7 @@ "js-types": ["js-types@4.0.0", "", {}, "sha512-/c+n06zvqFQGxdz1BbElF7S3nEghjNchLN1TjQnk2j10HYDaUc57rcvl6BbnziTx8NQmrg0JOs/iwRpvcYaxjQ=="], - "jsdoc-type-pratt-parser": ["jsdoc-type-pratt-parser@9.1.2", "", { "dependencies": { "@types/estree": "^1.0.9" } }, "sha512-9EXymowgk1mb9RY1VxuwKc+AhaxfBk2CV0dWxgGM+l5RURTtiUoAx7MlKwcsiVcEXK5HEPa7FeH/tsRpqjEPRg=="], + "jsdoc-type-pratt-parser": ["jsdoc-type-pratt-parser@9.2.0", "", { "dependencies": { "@types/estree": "^1.0.9" } }, "sha512-9TsacmMHRpB1pcKGuXZCJxV6MzYD6h0filpYuYnoO1WzUVin5dg8TFrwbfZSR3R6wlNN+Q81iwcIR2gzpCWmdg=="], "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], @@ -774,7 +774,7 @@ "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], + "picomatch": ["picomatch@4.0.7", "", {}, "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA=="], "pluralize": ["pluralize@8.0.0", "", {}, "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA=="], @@ -1008,7 +1008,7 @@ "supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], - "svgo": ["svgo@4.0.2", "", { "dependencies": { "commander": "^11.1.0", "css-select": "^5.1.0", "css-tree": "^3.0.1", "css-what": "^6.1.0", "csso": "^5.0.5", "picocolors": "^1.1.1", "sax": "^1.5.0" }, "bin": "./bin/svgo.js" }, "sha512-ekx94z1rRc5LDi6oSUaeRnYhd0UOJxdtQCL2rF8xpWxD3TPAsISWOrxezqGovqS38GRZOdpDfvQe3ts6F7nsng=="], + "svgo": ["svgo@4.1.0", "", { "dependencies": { "commander": "^11.1.0", "css-select": "^6.0.0", "css-tree": "^3.0.1", "css-what": "^7.0.0", "csso": "^5.0.5", "picocolors": "^1.1.1", "sax": "1.6.1" }, "bin": { "svgo": "bin/svgo.js" } }, "sha512-bkxnTg1kSU0guhIBmibA6UUhrQmPVA1XsQLN+ylCd+UWzbnLkySOcXpyk1mrl05f+pcaCx2eHb+sp6BgMZWX+Q=="], "synckit": ["synckit@0.11.13", "", { "dependencies": { "@pkgr/core": "^0.3.6" } }, "sha512-eNRKgb3z66Yp3D2CixVujOUvXLFUTij/zVnV8KRyvFdQwpz7I5DS8UfRkTeLzb64u+dkzDSdelE24izu+zSSUg=="], @@ -1134,7 +1134,7 @@ "@npmcli/promise-spawn/which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="], - "@types/concat-stream/@types/node": ["@types/node@26.2.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg=="], + "@types/concat-stream/@types/node": ["@types/node@26.3.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-L3fgrnchriRC2ExBflb8j4uZZURHZfQsmQeyVzhjcHW4kkwVyo8/0h1B2MVzMTrYUJYu6G7EWs14hW/L9putqw=="], "@typescript-eslint/visitor-keys/eslint-visitor-keys": ["eslint-visitor-keys@5.0.1", "", {}, "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA=="], diff --git a/codex/choices/choices_to_json.py b/codex/choices/choices_to_json.py index c5e5e04ce..1f5679fa7 100755 --- a/codex/choices/choices_to_json.py +++ b/codex/choices/choices_to_json.py @@ -25,6 +25,7 @@ from codex.choices.search import SEARCH_FIELDS from codex.choices.statii import ADMIN_STATUS_TITLES from codex.choices.tagging import ( + COUNTRIES, FORMAT_FIELD_SUPPORT, FORMAT_FIELD_VALUES, IDENTIFIER_SOURCES, @@ -67,6 +68,7 @@ "browser-table-column-costs.json": BROWSER_TABLE_COLUMN_COSTS, "browser-table-columns.json": BROWSER_TABLE_COLUMNS, "browser-table-default-columns.json": BROWSER_TABLE_DEFAULT_COLUMNS, + "countries.json": COUNTRIES, "format-field-support.json": FORMAT_FIELD_SUPPORT, "format-field-values.json": FORMAT_FIELD_VALUES, "identifier-sources.json": IDENTIFIER_SOURCES, diff --git a/codex/choices/tagging.py b/codex/choices/tagging.py index 1e6413db7..06671d5a0 100644 --- a/codex/choices/tagging.py +++ b/codex/choices/tagging.py @@ -58,6 +58,11 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType ) ) +# ISO 3166-1 countries (title=English name, value=2-letter code), sorted by name. +COUNTRIES = _vuetify_choices( + sorted((country.name, country.alpha_2) for country in pycountry.countries) +) + # comicbox identifier sources (Comic Vine, Metron, GCD, ...); title == value. IDENTIFIER_SOURCES = _vuetify_choices( (source.value, source.value) for source in IdSources @@ -78,7 +83,7 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType # most fields; the splits/renames are codex's relational model (issue split into # number + suffix so issues sort numerically, comicbox "arcs" stored as # story_arcs, ComicInfo's "manga"/"title" surfaced as reading_direction/stories). -# Canonical keys absent here (bookmark, date, pages, page_count, prices, +# Canonical keys absent here (bookmark, pages, page_count, prices, # updated_at, ...) are not tag-editor fields and are dropped. _CANONICAL_TO_EDITOR: MappingProxyType[str, tuple[str, ...]] = MappingProxyType( { @@ -87,6 +92,17 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType "series": ("series",), "volume": ("volume", "volume_issue_count"), "issue": ("issue_number", "issue_suffix"), + # An alternate numbering for the same issue, split like `issue`. + "alternative_issue": ( + "alternative_issue_number", + "alternative_issue_suffix", + ), + "collection_title": ("collection_title",), + # Only the parts codex stores (Comic.year/month/day). cover_date and + # store_date get no editor fields: codex keeps no columns for them, + # and comicbox derives cover_date -- MetronInfo's CoverDate -- from a + # full year/month/day trio on write. + "date": ("year", "month", "day"), "summary": ("summary",), "review": ("review",), "notes": ("notes",), @@ -103,7 +119,6 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType "monochrome": ("monochrome",), "original_format": ("original_format",), "manga": ("reading_direction",), - "reading_direction": ("reading_direction",), "credits": ("credits",), "language": ("language",), "age_rating": ("age_rating",), diff --git a/codex/librarian/fs/import_task.py b/codex/librarian/fs/import_task.py index 40f4b621c..830d988f3 100644 --- a/codex/librarian/fs/import_task.py +++ b/codex/librarian/fs/import_task.py @@ -48,8 +48,28 @@ def _remove_paths(kwargs: dict[str, Any], deleted_key: str, moved_key: str) -> N del kwargs[moved_key][src_path] +def _replaced_paths(kwargs: dict[str, Any], added_key: str, deleted_key: str) -> set: + """ + Take paths reported both deleted and added; they were replaced in place. + + An external tool that swaps a file by ``rm`` + ``mv``, or a watcher + backend that reports an atomic replace as a delete plus an add, leaves + both events in one batch. The recreated file carries a new inode, so + move detection can never pair them — and letting the delete win would + destroy the row, and its bookmarks, while a file sits at that very + path. It is the same path with new content: a modification. + """ + replaced = kwargs[added_key] & kwargs[deleted_key] + kwargs[added_key] -= replaced + kwargs[deleted_key] -= replaced + return replaced + + def _deduplicate(kwargs: dict[str, Any]) -> None: """Prune conflicting events on the same paths.""" + replaced_files = _replaced_paths(kwargs, "files_added", "files_deleted") + replaced_covers = _replaced_paths(kwargs, "covers_added", "covers_deleted") + # deleted wins over moved-from-this-source _remove_paths(kwargs, "dirs_deleted", "dirs_moved") _remove_paths(kwargs, "files_deleted", "files_moved") @@ -81,6 +101,11 @@ def _deduplicate(kwargs: dict[str, Any]) -> None: kwargs["covers_modified"] -= kwargs["covers_deleted"] kwargs["covers_modified"] -= kwargs["covers_added"] + # Added last: a replaced path is neither created nor deleted, and the + # subtractions above would have stripped it back out. + kwargs["files_modified"] |= replaced_files + kwargs["covers_modified"] |= replaced_covers + def build_import_task( library_id: int, diff --git a/codex/librarian/fs/mounted.py b/codex/librarian/fs/mounted.py new file mode 100644 index 000000000..25ed50d97 --- /dev/null +++ b/codex/librarian/fs/mounted.py @@ -0,0 +1,31 @@ +""" +Recognize a library root that isn't really there. + +A dropped network share, an ejected volume, or a docker bind mount that +didn't come up presents as an empty (or missing) directory rather than an +error. Every comic in the library then looks deleted at once, and acting +on that removes the rows and cascades their bookmarks away — for files +that are perfectly fine and will be back as soon as the mount is. + +The delete-phase existence check cannot help here: while the mount is +gone the files genuinely are unreachable. The only defense is to notice +the shape of the failure and refuse to act, which is what both scanners +do with this. +""" + +from pathlib import Path + +#: Docker bind mounts of a missing host path can be seeded with this file +#: so an unmounted volume is distinguishable from an empty library. +DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME" + + +def unmounted_reason(root: Path) -> str: + """Return why this library root looks unmounted, or "" if it looks fine.""" + if not root.is_dir(): + return "is not there" + if (root / DOCKER_UNMOUNTED_FN).exists(): + return "looks like an unmounted docker volume" + if not any(root.iterdir()): + return "is empty. Suspect unmounted" + return "" diff --git a/codex/librarian/fs/poller/poller.py b/codex/librarian/fs/poller/poller.py index b4c8277e3..ce4ec8845 100644 --- a/codex/librarian/fs/poller/poller.py +++ b/codex/librarian/fs/poller/poller.py @@ -10,6 +10,7 @@ from humanize import naturaldelta from codex.librarian.fs.import_task import build_import_task +from codex.librarian.fs.mounted import unmounted_reason from codex.librarian.fs.poller.snapshot import DatabaseSnapshot, DiskSnapshot from codex.librarian.fs.poller.snapshot_diff import SnapshotDiff from codex.librarian.fs.poller.status import FSPollStatus @@ -19,7 +20,6 @@ from codex.models import Library from codex.views.const import EPOCH_START -DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME" _DIR_NOT_FOUND_TIMEOUT = 15 * 60 _LIBRARY_ONLY = ( "path", @@ -67,32 +67,20 @@ def stop(self) -> None: # Timeout computation # ####################### - def _get_poll_timeout(self, library: Library) -> float | None: # noqa: PLR0911 + def _get_poll_timeout(self, library: Library) -> float | None: """ Compute seconds until this library's next scheduled poll. Returns None to wait forever (manual poll only). """ watch_path = Path(library.path) - unmounted_marker = watch_path / DOCKER_UNMOUNTED_FN if not library.poll: self.log.info(f"Library {library.path} waiting for manual poll.") return None - if not watch_path.is_dir(): - self.log.warning(f"Library {library.path} not found. Not polling.") - return _DIR_NOT_FOUND_TIMEOUT - - if unmounted_marker.exists(): - warning = f"Library {library.path} looks like an unmounted docker volume. Not polling." - self.log.warning(warning) - return _DIR_NOT_FOUND_TIMEOUT - - if not tuple(watch_path.iterdir()): - self.log.warning( - f"{library.path} is empty. Suspect unmounted. Not polling." - ) + if reason := unmounted_reason(watch_path): + self.log.warning(f"Library {library.path} {reason}. Not polling.") return _DIR_NOT_FOUND_TIMEOUT if library.update_in_progress: diff --git a/codex/librarian/fs/watcher/dirs.py b/codex/librarian/fs/watcher/dirs.py index 936940366..72e5ff6ca 100644 --- a/codex/librarian/fs/watcher/dirs.py +++ b/codex/librarian/fs/watcher/dirs.py @@ -57,23 +57,27 @@ def expand_dir_deleted(dir_path: str, library_pk: int, batch: ChangeBatch) -> No ) ) + # Only true children. A bare prefix match also claims every sibling whose + # name merely starts with this one ("/c/Batman" would delete "/c/Batman + # Beyond" and all its comics), so terminate the prefix with a separator. + child_prefix = dir_path.rstrip(os.sep) + os.sep + # Child folders child_folder_paths = Folder.objects.filter( - library_id=library_pk, path__startswith=dir_path + library_id=library_pk, path__startswith=child_prefix ).values_list("path", flat=True) for path in child_folder_paths: - if path != dir_path: - batch.dir_deleted.append( - ( - library_pk, - FSEvent(src_path=path, change=FSChange.deleted, is_directory=True), - ) + batch.dir_deleted.append( + ( + library_pk, + FSEvent(src_path=path, change=FSChange.deleted, is_directory=True), ) + ) # Child comics and failed imports for model in (Comic, FailedImport): child_paths = model.objects.filter( - library_id=library_pk, path__startswith=dir_path + library_id=library_pk, path__startswith=child_prefix ).values_list("path", flat=True) for path in child_paths: batch.deleted.append( diff --git a/codex/librarian/fs/watcher/events.py b/codex/librarian/fs/watcher/events.py index 560303ea1..5b272774e 100644 --- a/codex/librarian/fs/watcher/events.py +++ b/codex/librarian/fs/watcher/events.py @@ -7,6 +7,7 @@ 3. Move detection -> match delete+add pairs by inode within a batch """ +import os from pathlib import Path from watchfiles import Change @@ -67,7 +68,14 @@ def _process_change( def _find_library(library_paths: dict[str, int], file_path: str) -> int | None: """Find which library a changed path belongs to.""" for lib_path, pk in library_paths.items(): - if file_path.startswith(lib_path): + # Terminate the root with a separator before matching. Libraries may + # legitimately be siblings sharing a name prefix ("/c/comics" and + # "/c/comics-kids" — the admin serializer only rejects nesting), and a + # bare prefix match would file the second library's events under the + # first, importing its comics at paths outside their own library. + if file_path == lib_path or file_path.startswith( + lib_path.rstrip(os.sep) + os.sep + ): return pk return None diff --git a/codex/librarian/fs/watcher/move.py b/codex/librarian/fs/watcher/move.py index 30e0b3db2..1070e36d7 100644 --- a/codex/librarian/fs/watcher/move.py +++ b/codex/librarian/fs/watcher/move.py @@ -1,6 +1,9 @@ """Watchfiles Move detection.""" +import os +from dataclasses import dataclass from pathlib import Path +from stat import S_IFMT, S_ISDIR from loguru import logger @@ -10,8 +13,23 @@ from codex.models.comic import Comic from codex.models.paths import CustomCover -# stat field index for inode +# stat field indexes, as stored by WatchedPath.set_stat +_MODE_INDEX = 0 _INODE_INDEX = 1 +_SIZE_INDEX = 6 + + +@dataclass(frozen=True, slots=True) +class _DeletedEntry: + """A deleted path that could be the source of a move.""" + + index: int + library_pk: int + event: FSEvent + stat: list + # The same batch also reports this path as written, so its stored + # size predates that write. See ``_is_move_compatible``. + written: bool def _model_for_event(event: FSEvent): @@ -23,8 +41,8 @@ def _model_for_event(event: FSEvent): return Comic -def _get_db_inode(event: FSEvent, library_pk: int) -> int | None: - """Look up the inode for a path from the database stat field.""" +def _get_db_stat(event: FSEvent, library_pk: int) -> list | None: + """Return the stored stat for a path, when it carries a usable inode.""" model = _model_for_event(event) stat = ( model.objects.filter(library_id=library_pk, path=event.src_path) @@ -32,19 +50,57 @@ def _get_db_inode(event: FSEvent, library_pk: int) -> int | None: .first() ) if stat and len(stat) > _INODE_INDEX and stat[_INODE_INDEX]: - return stat[_INODE_INDEX] + return stat return None -def _get_disk_inode(path: str) -> int | None: - """Stat a path on disk and return its inode, or None.""" +def _get_disk_stat(path: str) -> os.stat_result | None: + """Stat a path on disk, or None when it can't be read.""" try: - p = Path(path) - return p.stat().st_ino + return Path(path).stat() except OSError: return None +def _is_move_compatible(entry: _DeletedEntry, disk_stat: os.stat_result) -> bool: + """ + Reject inode-match pairs that can't be a real rename. + + Ported from the poller's identically-named check (see + ``codex.librarian.fs.poller.snapshot_diff``), which the watcher needs + for the same reason and for one of its own. Stored inodes carry no + device, so a deleted path's inode can collide with an added path from + another mount; and a bulk CBR->CBZ conversion frees many inodes while + creating many files, so on an inode-reusing filesystem a new CBZ can + be handed the inode a *different* comic's CBR just released. Pairing + either re-paths one comic's row onto another comic's file. + + Two cheap sanity checks make the inode match load-bearing only when + it's plausibly a rename: + + - File type must match. A real rename never crosses ``stat()`` + file-type bits, so a mode mismatch is always a collision. + - For files, size must match too. Renames preserve size, and two + unrelated archives are vanishingly unlikely to share a byte count. + Directory ``st_size`` varies with entry count, so it is exempt. + + The size check compares against the size stored at import, so it only + holds while that is still current. A tagger that writes tags in place + and then renames — the flow ``build_import_task`` remaps modify + events for — changes the size before the rename, so a batch that also + reports the source as written waives the size check rather than + dropping a real pair. + """ + db_stat = entry.stat + db_mode = db_stat[_MODE_INDEX] if len(db_stat) > _MODE_INDEX else 0 + if db_mode and S_IFMT(db_mode) != S_IFMT(disk_stat.st_mode): + return False + if entry.written or S_ISDIR(disk_stat.st_mode): + return True + db_size = db_stat[_SIZE_INDEX] if len(db_stat) > _SIZE_INDEX else None + return db_size is None or db_size == disk_stat.st_size + + def _detect_one_move( add_idx: int, add_value: tuple[int, FSEvent], @@ -54,27 +110,29 @@ def _detect_one_move( matched_deleted, ) -> None: add_lib_pk, add_event = add_value - disk_inode = _get_disk_inode(add_event.src_path) - if not disk_inode: + disk_stat = _get_disk_stat(add_event.src_path) + if not disk_stat or not disk_stat.st_ino: return - match = deleted_by_inode.get(disk_inode) - if not match: + entry = deleted_by_inode.get(disk_stat.st_ino) + if not entry: return - del_idx, del_lib_pk, del_event = match # Only match within the same library - if add_lib_pk != del_lib_pk: + if add_lib_pk != entry.library_pk: return - is_dir = Path(add_event.src_path).is_dir() - is_cover = add_event.is_cover or del_event.is_cover + if not _is_move_compatible(entry, disk_stat): + return + + is_dir = S_ISDIR(disk_stat.st_mode) + is_cover = add_event.is_cover or entry.event.is_cover move_events.append( ( add_lib_pk, FSEvent( - src_path=del_event.src_path, + src_path=entry.event.src_path, change=FSChange.moved, dest_path=add_event.src_path, is_directory=is_dir, @@ -83,8 +141,26 @@ def _detect_one_move( ) ) matched_added.add(add_idx) - matched_deleted.add(del_idx) - del deleted_by_inode[disk_inode] + matched_deleted.add(entry.index) + del deleted_by_inode[disk_stat.st_ino] + + +def _index_deleted(batch: ChangeBatch) -> dict[int, _DeletedEntry]: + """Build inode -> deleted entry from the batch's deleted list.""" + written_paths = frozenset(event.src_path for _, event in batch.modified) + deleted_by_inode: dict[int, _DeletedEntry] = {} + for idx, (lib_pk, event) in enumerate(batch.deleted): + stat = _get_db_stat(event, lib_pk) + if not stat: + continue + deleted_by_inode[stat[_INODE_INDEX]] = _DeletedEntry( + index=idx, + library_pk=lib_pk, + event=event, + stat=stat, + written=event.src_path in written_paths, + ) + return deleted_by_inode def detect_moves(batch: ChangeBatch) -> list[tuple[int, FSEvent]]: @@ -94,13 +170,7 @@ def detect_moves(batch: ChangeBatch) -> list[tuple[int, FSEvent]]: Returns move events. Matched FSEvents are removed from batch.added and batch.deleted in place. """ - # Build inode -> (index, library_pk, event) from deleted list - deleted_by_inode: dict[int, tuple[int, int, FSEvent]] = {} - for idx, (lib_pk, event) in enumerate(batch.deleted): - inode = _get_db_inode(event, lib_pk) - if inode: - deleted_by_inode[inode] = (idx, lib_pk, event) - + deleted_by_inode = _index_deleted(batch) if not deleted_by_inode: return [] @@ -117,7 +187,7 @@ def detect_moves(batch: ChangeBatch) -> list[tuple[int, FSEvent]]: matched_added, matched_deleted, ) - # Remove matched entries from added and deleted (reverse order to keep indices valid) + # Drop the matched entries; the move events carry them now. batch.added = [ pair for idx, pair in enumerate(batch.added) if idx not in matched_added ] diff --git a/codex/librarian/fs/watcher/watcher.py b/codex/librarian/fs/watcher/watcher.py index 12b175e9f..52e8c5945 100644 --- a/codex/librarian/fs/watcher/watcher.py +++ b/codex/librarian/fs/watcher/watcher.py @@ -9,6 +9,7 @@ from codex.librarian.fs.filters import is_ignored_path, match_comic from codex.librarian.fs.import_task import build_import_task +from codex.librarian.fs.mounted import unmounted_reason from codex.librarian.fs.watcher.events import process_changes from codex.librarian.fs.watcher.status import FSWatcherRestartStatus from codex.librarian.threads import NamedThread @@ -127,8 +128,41 @@ def _process_changes(self, changes: set[tuple[Change, str]]) -> None: for library_pk, events in events_by_library.items(): task = build_import_task(library_pk, events) - if task is not None: - self.librarian_queue.put(task) + if task is None: + continue + if self._is_a_vanished_library(task): + continue + self.librarian_queue.put(task) + + def _is_a_vanished_library(self, task) -> bool: + """ + Whether this task's deletes are really an unmounted library. + + A dropped share or volume presents every comic in the library as + deleted at once. The poller refuses to scan a library in that + state; the watcher already holds the events, so it has to refuse + to act on them. Only deletes are worth checking — an add or a + modify against a missing mount can't do damage. + """ + if not (task.files_deleted or task.dirs_deleted or task.covers_deleted): + return False + root = self._library_root(task.library_id) + if root is None: + return False + reason = unmounted_reason(root) + if not reason: + return False + self.log.warning( + f"Library {root} {reason}. Ignoring the deletes it just reported." + ) + return True + + def _library_root(self, library_pk: int) -> Path | None: + """Return a watched library's root path.""" + for path, pk in self._library_paths.items(): + if pk == library_pk: + return Path(path) + return None def _get_extant_paths(self, paths: list[str]) -> list[str]: extant_paths = [] diff --git a/codex/librarian/onlinetag/outcome_stats.py b/codex/librarian/onlinetag/outcome_stats.py index 6d1029a51..c3493035f 100644 --- a/codex/librarian/onlinetag/outcome_stats.py +++ b/codex/librarian/onlinetag/outcome_stats.py @@ -18,6 +18,11 @@ resolution; :class:`~comicbox.events.FileError` marks comics that raised before finishing. +Alongside those file-level buckets, ``source_status_by_path`` folds the +per-source events (every one carries both ``path`` and ``source``) into a +"what did each source do with this comic" map, which the snapshot renders as +one status table column per source. + One scan runs on a single daemon thread and emits its events inline, so the tallies need no locking. """ @@ -28,7 +33,22 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING -from comicbox.events import AutoWritten, FileError, FileFinished, PromptDeferred +from comicbox.events import ( + AutoWritten, + FileError, + FileFinished, + NoMatch, + PromptDeferred, + SearchStarted, + Skipped, +) + +from codex.librarian.onlinetag.statuses import ( + IN_FLIGHT, + MATCHED, + NEEDS_REVIEW, + NO_MATCH, +) if TYPE_CHECKING: from pathlib import Path @@ -41,6 +61,7 @@ class OnlineTagOutcomeStats: """Accumulate per-comic online-tagging outcomes for an end-of-session summary.""" matched_source_by_path: dict[Path, list[str]] = field(default_factory=dict) + source_status_by_path: dict[Path, dict[str, str]] = field(default_factory=dict) written_paths: set[Path] = field(default_factory=set) no_change_paths: set[Path] = field(default_factory=set) deferred_paths: set[Path] = field(default_factory=set) @@ -48,14 +69,14 @@ class OnlineTagOutcomeStats: def record(self, event: Event) -> None: """Fold one comicbox event into the running tallies.""" + self._record_outcome(event) + self._record_source_status(event) + + def _record_outcome(self, event: Event) -> None: + """Fold one event into the file-level outcome buckets.""" match event: case AutoWritten(path=path, source=source) if path and source: - # Under merge_all_sources a comic can be auto-written by more - # than one source, so accumulate (deduped, order preserved) - # rather than letting the last event clobber the first. - sources = self.matched_source_by_path.setdefault(path, []) - if source not in sources: - sources.append(source) + self._add_matched_source(path, source) case FileFinished(path=path, outcome=outcome) if path: bucket = ( self.written_paths if outcome == "written" else self.no_change_paths @@ -68,6 +89,68 @@ def record(self, event: Event) -> None: case _: pass + def _record_source_status(self, event: Event) -> None: + """Fold one event into the per-comic, per-source status map.""" + match event: + case SearchStarted(path=path, source=source) if path and source: + self._set_source_status(path, source, IN_FLIGHT) + case AutoWritten(path=path, source=source) if path and source: + self._set_source_status(path, source, MATCHED) + # A source that found nothing above the confidence floor and one + # whose matcher declined both mean "this source did not tag it" — + # the same distinction the file-level pipeline already collapses. + case ( + NoMatch(path=path, source=source) | Skipped(path=path, source=source) + ) if path and source: + self._set_source_status(path, source, NO_MATCH) + case PromptDeferred(path=path, source=source) if path and source: + self._set_source_status(path, source, NEEDS_REVIEW) + case FileFinished(path=path) | FileError(path=path) if path: + self._clear_searching_sources(path) + case _: + pass + + def _add_matched_source(self, path: Path, source: str) -> None: + """ + Attribute a match to the source that won it, deduped in win order. + + Under merge_all_sources a comic can be auto-written by more than one + source, so accumulate rather than letting the last event clobber the + first. + """ + sources = self.matched_source_by_path.setdefault(path, []) + if source not in sources: + sources.append(source) + + def _set_source_status(self, path: Path, source: str, status: str) -> None: + """Record what one source is doing (or did) with one comic.""" + self.source_status_by_path.setdefault(path, {})[source] = status + + def _clear_searching_sources(self, path: Path) -> None: + """ + Drop still-searching cells once a comic is done. + + A source whose search raised is logged and skipped without an event, + so its ``in_flight`` cell would otherwise linger on a finished comic. + """ + cells = self.source_status_by_path.get(path) + if not cells: + return + for source, status in tuple(cells.items()): + if status == IN_FLIGHT: + del cells[source] + + def record_prefetch_match(self, path: Path, source: str) -> None: + """ + Record a comic matched from its stored id before the search pass. + + The prepass fetches by id outside the event-emitting session, so it + seeds by hand what an ``AutoWritten`` + ``FileFinished`` pair would. + """ + self.written_paths.add(path) + self._add_matched_source(path, source) + self._set_source_status(path, source, MATCHED) + @property def matched(self) -> int: """Comics that a source matched and wrote tags to.""" diff --git a/codex/librarian/onlinetag/session_manager.py b/codex/librarian/onlinetag/session_manager.py index 2ad566ac5..838b303b8 100644 --- a/codex/librarian/onlinetag/session_manager.py +++ b/codex/librarian/onlinetag/session_manager.py @@ -17,7 +17,15 @@ from time import monotonic from typing import TYPE_CHECKING, Any, cast -from comicbox.events import Event, PromptDeferred, RateLimited +from comicbox.events import ( + AutoWritten, + Event, + NoMatch, + PromptDeferred, + RateLimited, + SearchCompleted, + Skipped, +) from comicbox.exceptions import ComicboxError from comicbox.online_session import MatchMode, OnlineCredentials, OnlineSession from django.utils.timezone import now, timedelta @@ -37,8 +45,6 @@ set_pending_prompts, ) from codex.librarian.onlinetag.session_snapshot import ( - USER_MATCHED, - USER_SKIPPED, build_snapshot, clear_resolved_outcomes, clear_resume_state, @@ -53,6 +59,7 @@ SessionState, serialize_candidate, ) +from codex.librarian.onlinetag.statuses import USER_MATCHED, USER_SKIPPED from codex.librarian.onlinetag.stored_id_prepass import build_stored_id_map from codex.librarian.onlinetag.tag_pass_runner import TagPassRunner from codex.librarian.onlinetag.tasks import ( @@ -187,6 +194,19 @@ def _on_event(self, event: Event) -> None: lookup_status = self._pass_runner.lookup_status if lookup_status: self._mark_rate_limited(lookup_status, state, event) + # A source that reported an outcome just completed a request, so + # whatever wait it was serving is provably over. Deadlines expire + # on their own epoch anyway; this releases one early rather than + # leaving the strip counting down against a source already back at + # work. SearchStarted is deliberately absent — it fires *before* + # the request, which is exactly when a rate limit is hit. + case ( + SearchCompleted(source=source) + | AutoWritten(source=source) + | NoMatch(source=source) + | Skipped(source=source) + ) if source: + self._pass_runner.source_retry_at.pop(source, None) case PromptDeferred() if state is not None: self._persist_prompts(state) case _: @@ -210,7 +230,12 @@ def _mark_rate_limited( if event.source: self._pass_runner.source_retry_at[event.source] = retry_at.timestamp() else: + # No delay means the retry budget is spent: the source has stopped + # waiting and given up, so drop its countdown instead of leaving + # the strip claiming a retry that will never come. status.retry_at = None + if event.source: + self._pass_runner.source_retry_at.pop(event.source, None) # Push the completion estimate out by the wait plus the work still # left, so the total countdown doesn't sail past zero while stalled. remaining = ( @@ -219,7 +244,12 @@ def _mark_rate_limited( else 0 ) work = ( - estimate_seconds(remaining, state.match_mode, state.sources) + estimate_seconds( + remaining, + state.match_mode, + state.sources, + merge_all_sources=state.merge_all_sources, + ) if state is not None else 0.0 ) @@ -279,13 +309,14 @@ def _defer_prompt_response( state.answered_fingerprints.add(fingerprint) self.librarian_queue.put(ONLINE_TAG_PROMPT_TASK) pk = prompt.get("pk") + source = prompt.get("source") if item.action == "skip": - record_resolution(pk, USER_SKIPPED) + record_resolution(pk, USER_SKIPPED, source) self.log.info( f"Online tag: skipped prompt for {prompt.get('path')!r} mid-scan." ) return - record_resolution(pk, USER_MATCHED) + record_resolution(pk, USER_MATCHED, source) state.deferred_applies.append( (prompt, item.action, item.payload, item.chosen_volume_id) ) @@ -297,7 +328,7 @@ def _defer_skip_all(self, state: SessionState) -> None: return state.answered_fingerprints.update(prompts.keys()) for prompt in prompts.values(): - record_resolution(prompt.get("pk"), USER_SKIPPED) + record_resolution(prompt.get("pk"), USER_SKIPPED, prompt.get("source")) set_pending_prompts({}) self.librarian_queue.put(ONLINE_TAG_PROMPT_TASK) self.log.info(f"Online tag: skipped {len(prompts)} prompt(s) mid-scan.") @@ -323,7 +354,9 @@ def _merge_task(self, state: SessionState, task: Any) -> None: new_paths = {} for comic in comics: path = Path(comic.path) - if path not in state.path_to_pk.values(): + # ``path_to_pk`` is keyed by path; its *values* are pks, which a + # Path never equals, so testing them admitted every comic twice. + if path not in state.path_to_pk: new_paths[path] = comic.pk if not new_paths: return @@ -483,10 +516,7 @@ def _prefetch_stored_ids( continue primary_source, tags = result batch[pk] = tags - state.stats.written_paths.add(path) - state.stats.matched_source_by_path.setdefault(path, []).append( - primary_source - ) + state.stats.record_prefetch_match(path, primary_source) if batch: self._commit_prefetch(state, comic_paths, batch) @@ -557,11 +587,17 @@ def run_session(self, task: BulkOnlineTagTask) -> None: with self._lock: self._sessions[task.session_id] = state set_active_scan_id(task.session_id) - # A fresh batch starts with a clean resolution record and resume - # descriptor so a prior batch's user_matched/user_skipped overlays — - # and any leftover paused remainder — don't bleed onto these comics. + # A fresh batch starts with a clean resolution record so a prior + # batch's user_matched/user_skipped overlays don't bleed onto these + # comics. clear_resolved_outcomes() - clear_resume_state() + # Record this batch's remainder up front rather than merely clearing + # the prior one (which overwriting does anyway). Publishing narrows it + # as comics finish, but the first publish is throttled and a daemon + # killed before it would otherwise leave the batch with no descriptor + # at all — unresumable, while the frozen snapshot still shows every + # comic queued. + set_resume_state(state.resume_params, list(state.path_to_pk.values())) start = monotonic() try: @@ -654,14 +690,15 @@ def resolve_prompt( remove_pending_prompt(fingerprint) self.librarian_queue.put(ONLINE_TAG_PROMPT_TASK) pk = prompt.get("pk") + source = prompt.get("source") if action == "skip": - record_resolution(pk, USER_SKIPPED) + record_resolution(pk, USER_SKIPPED, source) self.log.info(f"Online tag: skipped prompt for {prompt.get('path')!r}.") return # Recorded as user-matched up front; if the apply drifts it re-queues a # fresh prompt, which the read-time overlay shows as needs-review again # (the live prompt set wins over the recorded outcome). - record_resolution(pk, USER_MATCHED) + record_resolution(pk, USER_MATCHED, source) self._apply_resolution(prompt, action, payload, chosen_volume_id) def skip_all_prompts(self) -> int: @@ -670,7 +707,7 @@ def skip_all_prompts(self) -> int: count = len(prompts) if count: for prompt in prompts.values(): - record_resolution(prompt.get("pk"), USER_SKIPPED) + record_resolution(prompt.get("pk"), USER_SKIPPED, prompt.get("source")) set_pending_prompts({}) self.librarian_queue.put(ONLINE_TAG_PROMPT_TASK) self.log.info(f"Online tag: skipped {count} prompt(s).") diff --git a/codex/librarian/onlinetag/session_snapshot.py b/codex/librarian/onlinetag/session_snapshot.py index 214273256..bb0e3c124 100644 --- a/codex/librarian/onlinetag/session_snapshot.py +++ b/codex/librarian/onlinetag/session_snapshot.py @@ -22,9 +22,14 @@ review, error) and upcoming (queued) come first; finished rows fill the rest. ``comic_count`` always carries the true total so the UI can say "showing N of M". +- **Each row also carries per-source cells** (``source_statuses``): what each + source did with that comic, which the status table renders as one column per + source. They come from the same fold; only the rate-limit "waiting" state is + projected here, since it lives on the scan rather than the event stream. - **Status string values are deliberately snake_case** and pass through the camelCase API renderer untouched (it only camelizes dict *keys*); the - frontend matches on these literals. + frontend matches on these literals. They are defined in + :mod:`~codex.librarian.onlinetag.statuses` and re-exported here. - Lives in ``caches["tagging"]`` like the prompts/scan-id state, with no TTL. The active flag flips to False when the scan finishes so the final tally stays visible until the next batch starts. @@ -38,6 +43,22 @@ from codex.librarian.onlinetag.estimate import SOURCE_RATE_PER_MINUTE from codex.librarian.onlinetag.session_cache import get_pending_prompts +# Re-exported so callers can keep reading the status vocabulary off the module +# that renders it. +from codex.librarian.onlinetag.statuses import ( + ACTIONABLE, + ERROR, + FINISHED, + IN_FLIGHT, + LIVE_SOURCE_STATUSES, + MATCHED, + NEEDS_REVIEW, + NO_MATCH, + QUEUED, + USER_MATCHED, + WAITING, +) + if TYPE_CHECKING: from codex.librarian.onlinetag.session_state import SessionState @@ -45,25 +66,6 @@ _NO_TIMEOUT = None _MAX_COMIC_ROWS: Final = 500 -# Per-comic status values. These are emitted to the frontend verbatim (the -# camelCase renderer rewrites keys, never values), so the frontend matches on -# these exact strings. -QUEUED: Final = "queued" -IN_FLIGHT: Final = "in_flight" -MATCHED: Final = "matched" -NO_MATCH: Final = "no_match" -NEEDS_REVIEW: Final = "needs_review" -ERROR: Final = "error" -# Outcomes of admin match-review actions. A scan never produces these; they are -# overlaid at read time from the resolution record so a comic the admin picked -# or skipped no longer reads as still "needs review". -USER_MATCHED: Final = "user_matched" -USER_SKIPPED: Final = "user_skipped" - -# Display ordering buckets: actionable first, then upcoming, then finished. -_ACTIONABLE: Final = (IN_FLIGHT, NEEDS_REVIEW, ERROR) -_FINISHED: Final = (MATCHED, NO_MATCH) - _RESOLVED_KEY = "onlinetag:resolved_outcomes" _RESUME_KEY = "onlinetag:resume_state" @@ -96,7 +98,8 @@ def deactivate_snapshot() -> None: snapshot can still carry an ``in_flight`` row — dishonest once no scan is running. Demote it to ``queued`` (the batch tally already counts in-flight as queued, so no count changes) so the comic reads as still-to-do, which - is also exactly what Resume will re-run. + is also exactly what Resume will re-run. Per-source cells claiming a live + lookup or a rate-limit wait go the same way. """ snapshot = get_snapshot() if snapshot and snapshot.get("active"): @@ -104,9 +107,34 @@ def deactivate_snapshot() -> None: for comic in snapshot.get("comics") or []: if comic.get("status") == IN_FLIGHT: comic["status"] = QUEUED + _drop_live_source_statuses(comic) + _drop_rate_limits(snapshot) set_snapshot(snapshot) +def _drop_live_source_statuses(comic: dict[str, Any]) -> None: + """Strip a row's searching/waiting cells (nothing is live any more).""" + cells = comic.get("source_statuses") + if not cells: + return + for source, status in tuple(cells.items()): + if status in LIVE_SOURCE_STATUSES: + del cells[source] + + +def _drop_rate_limits(snapshot: dict[str, Any]) -> None: + """ + Disarm the sources strip's retry countdowns. + + A crash skips the pass runner's ``finally``, so a scan killed mid-wait can + freeze a retry deadline that is still in the future — leaving the strip + counting down, then stuck on "retrying…", for a scan that is not running. + """ + for source in snapshot.get("sources") or []: + source["rate_limited"] = False + source["retry_at_epoch"] = None + + # --- resolution outcomes ----------------------------------------------------- # # A scan freezes its snapshot when it finishes, but the admin can keep @@ -117,12 +145,12 @@ def deactivate_snapshot() -> None: # when a new scan starts; pruned for vanished comics by the janitor. -def get_resolved_outcomes() -> dict[int, str]: - """Return the {pk: user_matched|user_skipped} resolution record.""" +def get_resolved_outcomes() -> dict[int, Any]: + """Return the {pk: {status, sources}} resolution record.""" return cache.get(_RESOLVED_KEY, {}) or {} -def set_resolved_outcomes(outcomes: dict[int, str]) -> None: +def set_resolved_outcomes(outcomes: dict[int, Any]) -> None: """Replace the resolution record, or clear it when empty.""" if outcomes: cache.set(_RESOLVED_KEY, outcomes, timeout=_NO_TIMEOUT) @@ -130,12 +158,38 @@ def set_resolved_outcomes(outcomes: dict[int, str]) -> None: cache.delete(_RESOLVED_KEY) -def record_resolution(pk: int | None, status: str) -> None: - """Record one comic's match-review outcome (no-op without a pk).""" +def _normalize_resolution(value: Any) -> dict[str, Any]: + """ + Read one resolution record as {status, sources}. + + Records written before per-source columns are bare status strings; the + file-backed cache outlives an upgrade, so they still have to overlay — + just without a source to attribute them to. + """ + if isinstance(value, str): + return {"status": value, "sources": {}} + if isinstance(value, dict): + return { + "status": value.get("status") or "", + "sources": dict(value.get("sources") or {}), + } + return {"status": "", "sources": {}} + + +def record_resolution(pk: int | None, status: str, source: str | None = None) -> None: + """Record one comic's match-review outcome by source (no-op without a pk).""" if pk is None: return outcomes = get_resolved_outcomes() - outcomes[pk] = status + record = _normalize_resolution(outcomes.get(pk)) + if source: + record["sources"][source] = status + # Under merge_all_sources one comic can raise a prompt per source, so the + # file-level status is a reduction: a match from either source is the + # outcome that matters and outranks the other's skip. + seen = (*record["sources"].values(), record["status"], status) + record["status"] = USER_MATCHED if USER_MATCHED in seen else status + outcomes[pk] = record set_resolved_outcomes(outcomes) @@ -146,27 +200,35 @@ def clear_resolved_outcomes() -> None: def overlay_resolutions( snapshot: dict[str, Any], - review_pks: set, - resolved_outcomes: dict[int, str], + review_sources_by_pk: dict[int, tuple[str, ...]], + resolved_outcomes: dict[int, Any], ) -> dict[str, Any]: """ Reconcile a stored snapshot's per-comic statuses with current state. A comic still awaiting a prompt wins as ``needs_review`` (the live cache is authoritative over the frozen scan); otherwise a recorded resolution - replaces a stale status with ``user_matched`` / ``user_skipped``. The - needs-review tally is refreshed from the live prompt set. Mutates the - passed dict (a fresh deserialized copy from the cache) and returns it. + replaces a stale status with ``user_matched`` / ``user_skipped``. Both land + in the source column that prompted as well as on the row. The needs-review + tally is refreshed from the live prompt set. Mutates the passed dict (a + fresh deserialized copy from the cache) and returns it. """ for comic in snapshot.get("comics") or []: pk = comic.get("pk") - if pk in review_pks: + # Snapshots cached before per-source columns have no cells to update. + cells = comic.setdefault("source_statuses", {}) + if pk in review_sources_by_pk: comic["status"] = NEEDS_REVIEW + for source in review_sources_by_pk[pk]: + cells[source] = NEEDS_REVIEW elif pk in resolved_outcomes: - comic["status"] = resolved_outcomes[pk] + record = _normalize_resolution(resolved_outcomes[pk]) + if record["status"]: + comic["status"] = record["status"] + cells.update(record["sources"]) batch = snapshot.get("batch") if isinstance(batch, dict): - batch["needs_review"] = len(review_pks) + batch["needs_review"] = len(review_sources_by_pk) return snapshot @@ -237,8 +299,41 @@ def _comic_status(path, pk, stats, review_pks: set, *, in_flight: bool) -> str: return IN_FLIGHT if in_flight else QUEUED +def _row_source_statuses( + path, + status: str, + stats, + sources: tuple[str, ...], + waiting_sources: frozenset[str], +) -> dict[str, str]: + """Report what each source did with one comic, as its own column cell.""" + if status == QUEUED: + # Nothing has touched it yet; the row reads as queued in every column. + return {} + if status == ERROR: + # A worker exception is usually archive-level rather than any one + # source's doing, so every column reports it instead of mis-blaming. + return dict.fromkeys(sources, ERROR) + cells = dict(stats.source_status_by_path.get(path, {})) + if status == IN_FLIGHT: + # A rate-limit wait stalls the comic being looked up right now. Mark + # the throttled source even with no search cell of its own: id-fetch + # and series-cache wins emit no SearchStarted but can still hit it. + for source in sources: + if source in waiting_sources: + cells[source] = WAITING + return cells + # A cancelled scan never emits the FileFinished that would have cleared a + # source's searching cell, and a frozen row must not claim a live lookup. + return { + source: cell + for source, cell in cells.items() + if cell not in LIVE_SOURCE_STATUSES + } + + def _build_comic_rows( - state: SessionState, review_pks: set + state: SessionState, review_pks: set, waiting_sources: frozenset[str] ) -> tuple[list[dict[str, Any]], dict[str, int]]: """Build per-comic rows in processing order plus a status tally.""" stats = state.stats @@ -256,26 +351,31 @@ def _build_comic_rows( if status == IN_FLIGHT: in_flight_taken = True counts[status] += 1 - # A matched comic carries every source that auto-wrote it (one under - # first-wins, possibly several under merge_all_sources); empty when the - # win came without a source attribution (e.g. a failed-fetch edge). - won = stats.matched_source_by_path.get(path, ()) if status == MATCHED else () rows.append( - {"pk": pk, "path": str(path), "status": status, "won_sources": list(won)} + { + "pk": pk, + "path": str(path), + "status": status, + "source_statuses": _row_source_statuses( + path, status, stats, state.sources, waiting_sources + ), + } ) return rows, counts def _order_and_cap(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: """Actionable rows first, then queued, then finished; capped, order kept.""" - actionable = [r for r in rows if r["status"] in _ACTIONABLE] + actionable = [r for r in rows if r["status"] in ACTIONABLE] queued = [r for r in rows if r["status"] == QUEUED] - finished = [r for r in rows if r["status"] in _FINISHED] + finished = [r for r in rows if r["status"] in FINISHED] return (actionable + queued + finished)[:_MAX_COMIC_ROWS] def _build_sources( - state: SessionState, source_retry_at: dict[str, float], now_epoch: float + state: SessionState, + source_retry_at: dict[str, float], + waiting_sources: frozenset[str], ) -> list[dict[str, Any]]: """One ordered entry per source: rate budget + any live retry countdown.""" # Live per-account budget (comicbox>=4.3.0 reads it off Metron's @@ -287,7 +387,7 @@ def _build_sources( sources = [] for source in state.sources: retry_at = source_retry_at.get(source) - rate_limited = bool(retry_at and retry_at > now_epoch) + rate_limited = source in waiting_sources sustained = (live.get(source) or {}).get("sustained") or {} sources.append( { @@ -313,7 +413,14 @@ def build_snapshot( ) -> dict[str, Any]: """Fold scan state + pending prompts into a JSON-safe snapshot dict.""" review_pks = {p.get("pk") for p in get_pending_prompts().values()} - rows, counts = _build_comic_rows(state, review_pks) + # One predicate for both the sources strip and the per-comic waiting cells, + # so a source can never read as limited in one and free in the other. + waiting_sources = frozenset( + source + for source, retry_at in source_retry_at.items() + if retry_at and retry_at > now_epoch + ) + rows, counts = _build_comic_rows(state, review_pks, waiting_sources) total = state.total_comics or len(state.path_to_pk) batch = { "total": total, @@ -337,7 +444,7 @@ def build_snapshot( # the frontend shows "Tagging" regardless; this matters once inactive. "resumable": bool(batch["queued"]), "batch": batch, - "sources": _build_sources(state, source_retry_at, now_epoch), + "sources": _build_sources(state, source_retry_at, waiting_sources), "comics": shown, "comic_count": len(rows), "shown_count": len(shown), diff --git a/codex/librarian/onlinetag/statuses.py b/codex/librarian/onlinetag/statuses.py new file mode 100644 index 000000000..3368ef5ae --- /dev/null +++ b/codex/librarian/onlinetag/statuses.py @@ -0,0 +1,38 @@ +""" +The per-comic status vocabulary for an online tagging scan. + +These strings are the wire contract with the admin Tagging tab's status +table: they reach the frontend verbatim (the camelCase renderer rewrites dict +*keys*, never values), so the frontend matches on these exact literals. + +They live in their own dependency-free module because both ends of the +pipeline need them — :mod:`~codex.librarian.onlinetag.outcome_stats` folds +events into them and :mod:`~codex.librarian.onlinetag.session_snapshot` +renders them — and importing one from the other would close a cycle. +""" + +from __future__ import annotations + +from typing import Final + +QUEUED: Final = "queued" +IN_FLIGHT: Final = "in_flight" +MATCHED: Final = "matched" +NO_MATCH: Final = "no_match" +NEEDS_REVIEW: Final = "needs_review" +ERROR: Final = "error" +# Per-source only: this source is throttled and the lookup is sitting out its +# retry wait. The file-level status stays in_flight — the comic *is* being +# worked on, just not right this second. +WAITING: Final = "waiting" +# Outcomes of admin match-review actions. A scan never produces these; they are +# overlaid at read time from the resolution record so a comic the admin picked +# or skipped no longer reads as still "needs review". +USER_MATCHED: Final = "user_matched" +USER_SKIPPED: Final = "user_skipped" + +# Display ordering buckets: actionable first, then upcoming, then finished. +ACTIONABLE: Final = (IN_FLIGHT, NEEDS_REVIEW, ERROR) +FINISHED: Final = (MATCHED, NO_MATCH) +# Per-source cell values that only make sense while a scan is running. +LIVE_SOURCE_STATUSES: Final = (IN_FLIGHT, WAITING) diff --git a/codex/librarian/onlinetag/tag_pass_runner.py b/codex/librarian/onlinetag/tag_pass_runner.py index c9f4c31ff..56ac4817d 100644 --- a/codex/librarian/onlinetag/tag_pass_runner.py +++ b/codex/librarian/onlinetag/tag_pass_runner.py @@ -45,9 +45,12 @@ def __init__( self.rate_limited: bool = False self.lookup_status: OnlineLookupStatus | None = None # source -> absolute epoch seconds the source is rate-limited until. - # Written by the manager on RateLimited events; cleared the moment a - # result arrives (the wait is provably over). The session snapshot - # reads it to drive per-source retry countdowns. + # Written by the manager on RateLimited events, which fire once per + # retry attempt just before the sleep — so an entry expires exactly + # when its wait ends, and readers filter by epoch rather than needing + # a clearing event. Cleared wholesale only at pass start and end, when + # nothing can be waiting. The session snapshot reads it to drive + # per-source retry countdowns. self.source_retry_at: dict[str, float] = {} def _flush_batch(self, state: SessionState, batch: dict[int, dict]) -> None: @@ -93,13 +96,15 @@ def _advance_result_status( state.completed_comics += 1 status.complete = state.completed_comics status.total = state.total_comics - # A yielded result means the wait (if any) is over: clear the - # rate-limit subtitle and the retry countdown, and re-estimate. + # A yielded result means the *scan* isn't stalled, so the global + # rate-limit subtitle and countdown clear. Per-source deadlines are + # deliberately left alone: they belong to a source, not to this comic, + # and one source finishing a comic says nothing about another's wait. + # They expire on their own epoch (see source_retry_at). status.subtitle = "" status.retry_at = None self._update_eta(state, status) self.rate_limited = False - self.source_retry_at.clear() self.status_controller.update(status) self._publish_snapshot(state) @@ -186,6 +191,14 @@ def collect_results( # frozen on its last "rate limited" subtitle — the librarian # thread recovers but the admin status bar would otherwise show # the wait forever, indistinguishable from a hang. + # + # The per-source deadlines go the same way, and for the same + # reason: a pause during a rate-limit wait aborts the sleep with + # its deadline still in the future, and run_session then freezes + # the snapshot the admin keeps looking at. Without this the paused + # table counts down and sticks on "retrying…" for a scan that is + # running nothing. self.lookup_status = None self.rate_limited = False + self.source_retry_at.clear() self.status_controller.finish(status) diff --git a/codex/librarian/scribe/importer/delete/__init__.py b/codex/librarian/scribe/importer/delete/__init__.py index 74eeb9330..fcd7e2334 100644 --- a/codex/librarian/scribe/importer/delete/__init__.py +++ b/codex/librarian/scribe/importer/delete/__init__.py @@ -11,12 +11,20 @@ def delete(self) -> None: """Delete files and folders.""" if self.abort_event.is_set(): return - self.counts.folders_deleted += self.bulk_folders_deleted() + folders_deleted, comics_cascaded, folder_collections = ( + self.bulk_folders_deleted() + ) + self.counts.folders_deleted += folders_deleted + # Comics under a deleted folder die by cascade, not by path, so they + # never reach ``bulk_comics_deleted`` to be counted there. + self.counts.comics_deleted += comics_cascaded if self.abort_event.is_set(): return - self.counts.comics_deleted, deleted_comic_collections = ( - self.bulk_comics_deleted() - ) + comics_deleted, deleted_comic_collections = self.bulk_comics_deleted() + self.counts.comics_deleted += comics_deleted + for model, pks in folder_collections.items(): + if pks: + deleted_comic_collections.setdefault(model, set()).update(pks) if self.abort_event.is_set(): return self.counts.covers_deleted = self.bulk_covers_deleted() diff --git a/codex/librarian/scribe/importer/delete/comics.py b/codex/librarian/scribe/importer/delete/comics.py index 68bd3512a..1087794a6 100644 --- a/codex/librarian/scribe/importer/delete/comics.py +++ b/codex/librarian/scribe/importer/delete/comics.py @@ -2,6 +2,7 @@ from codex.librarian.scribe.importer.const import ALL_COMIC_COLLECTION_FIELD_NAMES from codex.librarian.scribe.importer.delete.covers import DeletedCoversImporter +from codex.librarian.scribe.importer.delete.existence import confirm_deleted from codex.librarian.scribe.importer.statii.delete import ImporterRemoveComicsStatus from codex.models import Comic, Folder, StoryArc from codex.settings import ( @@ -9,6 +10,12 @@ IMPORTER_LINK_FK_BATCH_SIZE, ) +# A delete this large, and this much of the library, reads like a vanished +# mount rather than a user tidying up. The floor keeps small libraries from +# tripping it whenever a couple of comics are removed. +_MASS_DELETE_FLOOR = 50 +_MASS_DELETE_FRACTION = 0.5 + class DeletedComicsImporter(DeletedCoversImporter): """Delete comics methods.""" @@ -54,6 +61,26 @@ def _populate_deleted_comic_collections( ): cls._populate_deleted_comic_collection(deleted_comic_collections, comic) + def _warn_on_mass_delete(self, num_deleted: int) -> None: + """ + Flag a delete large enough to look like the library vanished. + + An unmounted volume or dropped network share makes every path read + as missing, which the existence backstop cannot tell from a real + mass deletion. Nothing is blocked here — this only leaves a + breadcrumb in the log for a user asking where their comics went. + """ + if num_deleted < _MASS_DELETE_FLOOR: + return + total = Comic.objects.filter(library=self.library).count() + if total and num_deleted >= total * _MASS_DELETE_FRACTION: + reason = ( + f"Deleting {num_deleted} of {total} comics in" + f" {self.library.path}. If that library lives on a network" + f" share or removable volume, check that it is still mounted." + ) + self.log.warning(reason) + def bulk_comics_deleted(self, **kwargs) -> tuple[int, dict]: """Bulk delete comics found missing from the filesystem.""" count = 0 @@ -64,8 +91,11 @@ def bulk_comics_deleted(self, **kwargs) -> tuple[int, dict]: return count, deleted_comic_collections self.status_controller.start(status) # Batch path__in to stay under SQLite's variable limit. - paths = tuple(self.task.files_deleted) + paths = confirm_deleted(self.task.files_deleted, self.log, "comics") self.task.files_deleted = frozenset() + if not paths: + return count, deleted_comic_collections + self._warn_on_mass_delete(len(paths)) delete_comic_pks: set[int] = set() for start in range(0, len(paths), IMPORTER_LINK_FK_BATCH_SIZE): if self.abort_event.is_set(): diff --git a/codex/librarian/scribe/importer/delete/covers.py b/codex/librarian/scribe/importer/delete/covers.py index 61c3f4739..1e590efad 100644 --- a/codex/librarian/scribe/importer/delete/covers.py +++ b/codex/librarian/scribe/importer/delete/covers.py @@ -1,6 +1,7 @@ """Clean up covers from the db.""" from codex.librarian.covers.tasks import CoverRemoveTask +from codex.librarian.scribe.importer.delete.existence import confirm_deleted from codex.librarian.scribe.importer.search import SearchIndexImporter from codex.librarian.scribe.importer.statii.delete import ImporterRemoveCoversStatus from codex.models.paths import CustomCover @@ -23,10 +24,11 @@ def bulk_covers_deleted(self, **kwargs) -> int: if not self.task.covers_deleted: return 0 self.status_controller.start(status) - covers = CustomCover.objects.filter( - library=self.library, path__in=self.task.covers_deleted - ) + paths = confirm_deleted(self.task.covers_deleted, self.log, "covers") self.task.covers_deleted = frozenset() + if not paths: + return 0 + covers = CustomCover.objects.filter(library=self.library, path__in=paths) delete_cover_pks = frozenset(covers.values_list("pk", flat=True)) count, _ = covers.delete() diff --git a/codex/librarian/scribe/importer/delete/existence.py b/codex/librarian/scribe/importer/delete/existence.py new file mode 100644 index 000000000..898416f0a --- /dev/null +++ b/codex/librarian/scribe/importer/delete/existence.py @@ -0,0 +1,49 @@ +""" +Confirm a scanner's deletes against the filesystem before acting on them. + +Deleting a comic row cascades its bookmarks and read progress away, and +nothing restores them — the next scan re-imports the file as a fresh, +unread comic. So a delete is only safe when the file is really gone. + +Both scanners infer deletes rather than observing them, and every inference +they make has failure modes: a watcher batch that reports a delete whose +paired add lands in the *next* batch, a directory expansion that overmatched, +an inode pair the compatibility checks refused. In each case the path is +still on disk, and the delete is wrong. + +Probing the path is cheap next to what it protects, and a path that is +genuinely gone answers immediately. A row skipped here is not stranded: it +still points at a real file, so the next scan reconciles it normally — a +stale row costs a re-read, a wrongly deleted one costs the user's place in +the book. + +This cannot save a library whose whole mount disappeared, where every path +reads as missing. ``DeletedComicsImporter`` logs that case instead. +""" + +from collections.abc import Collection +from pathlib import Path + + +def split_extant(paths: Collection[str]) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Partition paths into (gone from disk, still on disk).""" + gone: list[str] = [] + extant: list[str] = [] + for path in paths: + if Path(path).exists(): + extant.append(path) + else: + gone.append(path) + return tuple(gone), tuple(extant) + + +def confirm_deleted(paths: Collection[str], log, kind: str) -> tuple[str, ...]: + """Return only the paths that are really gone, reporting any that aren't.""" + gone, extant = split_extant(paths) + if extant: + reason = ( + f"Not deleting {len(extant)} {kind} a scan reported missing that" + f" are still on disk. The next scan will reconcile them." + ) + log.warning(reason) + return gone diff --git a/codex/librarian/scribe/importer/delete/folders.py b/codex/librarian/scribe/importer/delete/folders.py index fd4307d3d..5245eda7b 100644 --- a/codex/librarian/scribe/importer/delete/folders.py +++ b/codex/librarian/scribe/importer/delete/folders.py @@ -1,6 +1,7 @@ """Delete database folders methods.""" from codex.librarian.scribe.importer.delete.comics import DeletedComicsImporter +from codex.librarian.scribe.importer.delete.existence import confirm_deleted from codex.librarian.scribe.importer.statii.delete import ImporterRemoveFoldersStatus from codex.models.collections import Folder from codex.models.comic import Comic @@ -9,26 +10,39 @@ class DeletedFoldersImporter(DeletedComicsImporter): """Delete database folders methods.""" - def bulk_folders_deleted(self, **kwargs) -> int: - """Bulk delete folders.""" + def bulk_folders_deleted(self, **kwargs) -> tuple[int, int, dict]: + """ + Bulk delete folders. Return (folders, cascaded comics, collections). + + Comics under a deleted folder die by the ``parent_folder`` cascade + rather than through ``bulk_comics_deleted``, so their collections + are gathered here too. Without them a series or publisher emptied by + a folder delete is never re-stamped, and browsers viewing it keep + listing comics that are gone until some unrelated import moves the + timestamp. + """ status = ImporterRemoveFoldersStatus(0, len(self.task.dirs_deleted)) + deleted_comic_collections = self._init_deleted_comic_collections() try: if not self.task.dirs_deleted: - return 0 + return 0, 0, deleted_comic_collections self.status_controller.start(status) - folders = Folder.objects.filter( - library=self.library, path__in=self.task.dirs_deleted - ) + paths = confirm_deleted(self.task.dirs_deleted, self.log, "folders") self.task.dirs_deleted = frozenset() - delete_comic_pks = frozenset( - Comic.objects.filter(library=self.library, folders__in=folders) - .distinct() - .values_list("pk", flat=True) + if not paths: + return 0, 0, deleted_comic_collections + folders = Folder.objects.filter(library=self.library, path__in=paths) + folder_count = folders.count() + delete_comic_qs = Comic.objects.filter( + library=self.library, folders__in=folders + ).distinct() + self._populate_deleted_comic_collections( + delete_comic_qs, deleted_comic_collections ) + delete_comic_pks = frozenset(delete_comic_qs.values_list("pk", flat=True)) folders.delete() - count = len(delete_comic_pks) self.remove_covers(delete_comic_pks, custom=False) finally: self.status_controller.finish(status) - return count + return folder_count, len(delete_comic_pks), deleted_comic_collections diff --git a/codex/librarian/scribe/importer/init.py b/codex/librarian/scribe/importer/init.py index eae9c875b..6cdf11065 100644 --- a/codex/librarian/scribe/importer/init.py +++ b/codex/librarian/scribe/importer/init.py @@ -385,14 +385,16 @@ def init_apply(self) -> None: """Initialize the library and status flags.""" self.start_time = now() self.library.start_update() - too_long = self._wait_for_filesystem_ops_to_finish() - if too_long: + if self._wait_for_filesystem_ops_to_finish(): + # The import runs anyway: abandoning the task would drop these + # events entirely on a watched library that isn't also polled. + # Files still mid-copy fail to import and are retried by a later + # scan, so say that rather than implying nothing was imported. reason = ( "Import apply waited for the filesystem to stop changing too long. " - "Try polling again once files have finished copying" - f" in library: {self.library.path}" + "Importing anyway; files still copying may fail and be retried" + f" on a later scan in library: {self.library.path}" ) self.log.warning(reason) - return self._log_task() self._init_librarian_status(self.library.path) diff --git a/codex/librarian/scribe/priority.py b/codex/librarian/scribe/priority.py index 1212ef2f9..e2725551f 100644 --- a/codex/librarian/scribe/priority.py +++ b/codex/librarian/scribe/priority.py @@ -1,6 +1,7 @@ """Priority for Scribe tasks in the PriorityQueue.""" from datetime import UTC, datetime +from itertools import count from codex.librarian.scribe.importer.tasks import ( ImportTask, @@ -77,9 +78,18 @@ JanitorDumpUserDataTask, ) +# Final element of every priority tuple. Tasks are pushed onto a heap as +# ``(priority, task)``; when two priorities compare equal the heap falls +# through to comparing the tasks themselves, and ScribeTask dataclasses +# define no ordering — a ``TypeError`` in the routing thread that loses the +# task. Timestamps tie more often than they look like they would (they are +# truncated, and a clock can step backwards), so carry a strictly monotonic +# counter that can never tie. ``next`` on an ``itertools.count`` is atomic. +_TIE_BREAKER = count() -def get_task_priority(task: ScribeTask) -> tuple[int, float]: + +def get_task_priority(task: ScribeTask) -> tuple[int, float, int]: """Get task priority by index.""" now = datetime.now(tz=UTC).timestamp() priority = _SCRIBE_TASK_PRIORITY.index(type(task)) - return priority, now + return priority, now, next(_TIE_BREAKER) diff --git a/codex/librarian/scribe/scribed.py b/codex/librarian/scribe/scribed.py index 7c898bed3..deb33f876 100644 --- a/codex/librarian/scribe/scribed.py +++ b/codex/librarian/scribe/scribed.py @@ -48,7 +48,13 @@ class ScribeThread(QueuedThread): """A worker to handle all bulk database updates.""" - SHUTDOWN_MSG = (0, QueuedThread.SHUTDOWN_MSG) + # Shaped like a real queue item — ``(get_task_priority(...), task)`` — + # because a PriorityQueue orders the shutdown message against whatever + # is already queued. A bare int here raises TypeError comparing int to + # tuple, so stopping the thread with a task pending would abort the + # daemon's shutdown loop. The negative index sorts it ahead of every + # task so a stop is honored promptly. + SHUTDOWN_MSG = ((-1, 0.0, -1), QueuedThread.SHUTDOWN_MSG) # Importer / janitor / search bursts are minutes-to-hours apart on # a typical install. Releasing the conn between bursts saves an # open file handle + ~50 KiB pinned for the entire idle gap; the diff --git a/codex/librarian/scribe/tag_writer.py b/codex/librarian/scribe/tag_writer.py index f3541ed85..58f577561 100644 --- a/codex/librarian/scribe/tag_writer.py +++ b/codex/librarian/scribe/tag_writer.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import TYPE_CHECKING, cast -from comicbox.box import Comicbox from comicbox.config import get_config from comicbox.events import ( BatchFinished, @@ -16,16 +15,28 @@ FileParsed, FileShortCircuited, ) -from comicbox.formats import MetadataFormats from comicbox.write import BulkWriteItem, bulk_write +from django.core.cache import cache +from django.utils.timezone import now -from codex.librarian.notifier.tasks import TAG_WRITE_ERRORS_CHANGED_TASK +from codex.librarian.notifier.tasks import ( + LIBRARY_CHANGED_TASK, + TAG_WRITE_ERRORS_CHANGED_TASK, +) +from codex.librarian.scribe.importer.importer import ComicImporter from codex.librarian.scribe.importer.tasks import ImportTask from codex.librarian.scribe.status import TagWriteStatus from codex.librarian.scribe.tagwrite_errors import add_tag_write_error +from codex.librarian.scribe.tagwrite_rename import ( + RenamePlan, + build_predict_config, + plan_rename, + predict_name, + will_convert, +) +from codex.librarian.scribe.timestamp_update import TimestampUpdater from codex.librarian.worker import WorkerStatusAbortableBase from codex.models.comic import Comic -from codex.settings import COMICBOX_CONFIG if TYPE_CHECKING: from comicbox.events import Event @@ -92,27 +103,6 @@ def _build_items( ) return items - def _reimport_unwatched(self, comic_paths: dict[int, Path]) -> None: - """Re-import comics in libraries without filesystem event watching.""" - if not comic_paths: - return - comics = Comic.objects.filter(pk__in=comic_paths.keys()).only( - "pk", "path", "library_id", "library__events" - ) - library_path_map: defaultdict[int, set[str]] = defaultdict(set) - for comic in comics: - if not comic.library.events: - library_path_map[comic.library_id].add(comic.path) # pyright: ignore[reportAttributeAccessIssue] - - for library_id, paths in library_path_map.items(): - import_task = ImportTask( - library_id=library_id, - files_modified=frozenset(paths), - force_import_metadata=True, - check_metadata_mtime=False, - ) - self.librarian_queue.put(import_task) - @staticmethod def _build_base_config(task: BulkTagWriteTask): """ @@ -129,14 +119,21 @@ def _build_base_config(task: BulkTagWriteTask): cfg = get_config() return replace(cfg, general=replace(cfg.general, delete_orig=True)) - def _collect_written_pks( + def _collect_written_paths( self, items: list[BulkWriteItem], path_to_pk: dict[Path, int], base_config, - ) -> set[int]: - """Run bulk_write and return pks that were successfully written.""" - written_pks: set[int] = set() + ) -> dict[int, Path]: + """ + Run bulk_write; map successfully written pks to their on-disk paths. + + The mapped path is the *final* one comicbox reports: writing an + unwritable archive (CBR/CBT/CB7) repacks it as a CBZ at a new path, + and every later step — rename, DB sync — must chase the file there, + not the submitted path the DB still holds. + """ + written_paths: dict[int, Path] = {} had_errors = False for result in bulk_write( items, @@ -153,167 +150,378 @@ def _collect_written_pks( continue pk = path_to_pk.get(result.path) if pk is not None: - written_pks.add(pk) + written_paths[pk] = result.final_path or result.path if had_errors: # Surface the failures to admins (red badge + Tagging-tab panel). self.librarian_queue.put(TAG_WRITE_ERRORS_CHANGED_TASK) - return written_pks + return written_paths @staticmethod def _resolve_comics( task: BulkTagWriteTask, - ) -> tuple[dict[int, Path], dict[int, int], dict[int, bool]]: + ) -> tuple[dict[int, Path], dict[int, int]]: """ Resolve writable comics to path / library maps. Never returns comics in read-only libraries, even if a task somehow carries their pks (the API funnel already drops them; this is a - backstop). Returns (path-by-pk, library-id-by-pk, watcher-by-library). + backstop). Returns (path-by-pk, library-id-by-pk). + + Whether a library is watched no longer changes anything here: every + move is applied inline, and the re-read is requested either way + because a watched library's own re-read is stat-only unless the + import-metadata flag is on. """ comics = ( Comic.objects.filter(pk__in=task.comic_pks) .exclude(library__read_only=True) - .select_related("library") - .only("pk", "path", "library__events") + .only("pk", "path", "library") ) comic_paths: dict[int, Path] = {} lib_of: dict[int, int] = {} - library_events: dict[int, bool] = {} for comic in comics: comic_paths[comic.pk] = Path(comic.path) lib_of[comic.pk] = comic.library_id # pyright: ignore[reportAttributeAccessIssue] - library_events[comic.library_id] = comic.library.events # pyright: ignore[reportAttributeAccessIssue] - return comic_paths, lib_of, library_events + return comic_paths, lib_of def write_tags(self, task: BulkTagWriteTask) -> None: - """Execute bulk tag write, optional rename, and re-import.""" + """ + Rename each archive to its final name, then write tags there. + + Renaming first, and applying each move to the database before this + method returns, is what keeps a comic's bookmarks through the churn. + Every path a write moves through used to be reconciled *later*, by + an ``ImportTask`` queued behind whatever else the scribe was doing — + so a scan landing in between saw an unexplained delete plus create, + deleted the row by its now-dead path, and cascaded the bookmarks + away. Doing the move here, on the scribe's own thread, means no scan + can be processed while the database disagrees with the disk: a + stale delete finds no row and a stale create converges onto the row + already sitting at that path. + + A conversion (CBR/CBT/CB7 repacked as a CBZ) still moves the file + after the write, so it is synced the same way as soon as the batch + finishes. Only the metadata re-read is left to a queued task, which + is safe because it names a path the database already holds. + """ if not task.comic_pks: self.log.debug("Tag write called with no comic pks.") return - comic_paths, lib_of, library_events = self._resolve_comics(task) - items = self._build_items(task, comic_paths) - if not items and not task.rename: + comic_paths, lib_of = self._resolve_comics(task) + if not self._build_items(task, comic_paths) and not task.rename: self.log.debug("Tag write: no patches to apply.") return - written_pks: set[int] = set() - if items: - path_to_pk = {path: pk for pk, path in comic_paths.items()} - base_config = self._build_base_config(task) - written_pks = self._collect_written_pks(items, path_to_pk, base_config) - - renamed_pks: set[int] = set() - if task.rename: - # Rename-only (no patch) renames every resolved comic from its - # existing on-archive metadata; with a patch, only the written ones. - candidates = written_pks if items else set(comic_paths) - renamed_pks = self._rename_comics( - candidates, - comic_paths, - lib_of, - library_events, - tags_written=bool(items), - ) + renamed_paths = self._rename_first(task, comic_paths, lib_of) + current_paths = {**comic_paths, **renamed_paths} + + written_paths = self._write(task, current_paths) + converted_paths = self._sync_conversions( + task, current_paths, written_paths, lib_of + ) + # A kept original leaves the row where it is and the new CBZ is + # simply a new file, so its scheme name is applied after the write. + post_renamed = self._rename_kept_conversions( + task, current_paths, written_paths, converted_paths + ) - # Non-renamed written comics keep the existing unwatched re-import path; - # renamed comics are synced inside _rename_comics (their old path is gone). - non_renamed = { - pk: comic_paths[pk] for pk in written_pks if pk not in renamed_pks - } - self._reimport_unwatched(non_renamed) - self.log.info( - f"Tag write complete: {len(written_pks)} written, {len(renamed_pks)} renamed." + self._enqueue_rereads( + current_paths, written_paths, converted_paths, post_renamed, lib_of ) + num_renamed = len(renamed_paths) + len(post_renamed) + reason = ( + f"Tag write complete: {len(written_paths)} written, {num_renamed} renamed." + ) + self.log.info(reason) + + def _write( + self, task: BulkTagWriteTask, current_paths: dict[int, Path] + ) -> dict[int, Path]: + """Write tags at each comic's post-rename path.""" + items = self._build_items(task, current_paths) + if not items: + return {} + path_to_pk = {path: pk for pk, path in current_paths.items()} + base_config = self._build_base_config(task) + return self._collect_written_paths(items, path_to_pk, base_config) - def _rename_one(self, old_path: Path) -> Path | None: + def _plan_renames( + self, task: BulkTagWriteTask, comic_paths: dict[int, Path] + ) -> list[RenamePlan]: """ - Rename one archive to the comicbox (comicfn2dict) filename scheme. + Build every comic's rename plan, dropping the ones that can't run. - Returns the new path, or None when the name is unchanged or no name - could be built. Raises ``FileExistsError`` on a collision with a - *different* file so the caller reports it without clobbering anything - (comicbox's ``rename_file`` does a bare ``Path.rename``). + A plan is dropped when two comics in the batch predict the same + name, when either the rename target or the destination a later + conversion needs is already taken on disk or in the database, or + when no name could be built at all. Both ends matter: comicbox + refuses to convert onto an existing file, and that refusal would + land *after* the rename and its database move had already happened. """ - with Comicbox(old_path, config=COMICBOX_CONFIG) as car: - # to_string(FILENAME) is exactly what rename_file() derives the - # name from (schema.dumps(_to_dict(FILENAME))), so this pre-check - # targets the precise destination rename_file() will use. - target = car.to_string(MetadataFormats.FILENAME) - if not target: + config = build_predict_config(task.delete_keys, task.mode) + plans: list[RenamePlan] = [] + claimed: dict[Path, int] = {} + for pk, old_path in comic_paths.items(): + if not task.delete_original and will_convert(old_path): + # Renamed after the write instead; the row stays put. + continue + try: + plan = plan_rename(pk, old_path, self._patch_for(task, pk), config) + except Exception as exc: + self._report_error(old_path, f"rename failed: {exc}") + continue + if plan is None: self.log.warning(f"Rename skipped; no filename built for {old_path}") - return None - new_path = old_path.parent / target - if new_path == old_path: - return None - if new_path.exists() and not new_path.samefile(old_path): - reason = f"rename target already exists: {new_path}" - raise FileExistsError(reason) - car.rename_file() - renamed = car.get_path() - return renamed or new_path - - def _rename_comics( + continue + if plan.target == old_path and plan.final_path == old_path: + continue + if reason := self._claim_conflict(plan, claimed, comic_paths): + self._report_error(old_path, reason) + continue + claimed[plan.target] = pk + claimed[plan.final_path] = pk + plans.append(plan) + return plans + + @staticmethod + def _patch_for(task: BulkTagWriteTask, pk: int) -> dict | None: + """Return the patch this comic will be written with, if any.""" + if task.per_comic_patches and pk in task.per_comic_patches: + return task.per_comic_patches[pk] + return task.patch or None + + @staticmethod + def _destination_conflict( + plan: RenamePlan, destination: Path, claimed: dict[Path, int] + ) -> str: + """Return why one destination is unavailable, or "".""" + other = claimed.get(destination) + if other is not None and other != plan.pk: + return f"another comic in this batch renames to {destination}" + if destination == plan.old_path: + return "" + if destination.exists() and not ( + # A case-only rename on a case-insensitive filesystem finds + # itself at the destination; that is the file we are moving. + plan.old_path.exists() and destination.samefile(plan.old_path) + ): + return f"rename target already exists: {destination}" + if Comic.objects.filter(path=str(destination)).exclude(pk=plan.pk).exists(): + return f"another comic already holds {destination}" + return "" + + @classmethod + def _claim_conflict( + cls, plan: RenamePlan, claimed: dict[Path, int], comic_paths: dict[int, Path] + ) -> str: + """Return why this plan's destinations are unavailable, or "".""" + for destination in (plan.target, plan.final_path): + if reason := cls._destination_conflict(plan, destination, claimed): + return reason + # A path another comic in this batch is renaming *away* from is + # only free once that rename runs, which it may not. + for other_pk, other_path in comic_paths.items(): + if other_pk != plan.pk and other_path in (plan.target, plan.final_path): + return f"{plan.target} is another comic's current path" + return "" + + def _rename_first( self, - candidates: set[int], + task: BulkTagWriteTask, comic_paths: dict[int, Path], lib_of: dict[int, int], - library_events: dict[int, bool], - *, - tags_written: bool, - ) -> set[int]: - """Rename candidate comics and sync the DB. Return the renamed pks.""" - # library_id -> {old_path_str: new_path_str} - moved: defaultdict[int, dict[str, str]] = defaultdict(dict) - renamed_pks: set[int] = set() - had_errors = False - for pk in candidates: - old_path = comic_paths[pk] + ) -> dict[int, Path]: + """Rename archives to their scheme names and move the rows with them.""" + if not task.rename: + return {} + plans = self._plan_renames(task, comic_paths) + renamed: dict[int, Path] = {} + moves: defaultdict[int, dict[int, tuple[str, str]]] = defaultdict(dict) + for plan in plans: + if plan.target == plan.old_path: + continue + try: + plan.old_path.rename(plan.target) + except OSError as exc: + self._report_error(plan.old_path, f"rename failed: {exc}") + continue + renamed[plan.pk] = plan.target + moves[lib_of[plan.pk]][plan.pk] = ( + str(plan.old_path), + str(plan.target), + ) + for library_id, library_moves in moves.items(): + for pk in self._apply_moves_inline(library_id, library_moves): + # The database refused the move, so put the file back + # rather than leave the row pointing at a path that no + # longer exists. + self._revert_rename(pk, library_moves[pk]) + renamed.pop(pk, None) + return renamed + + def _revert_rename(self, pk: int, move: tuple[str, str]) -> None: + """Undo a disk rename whose database move did not take.""" + src, dest = move + old_path = Path(src) + new_path = Path(dest) + try: + if new_path.exists() and not old_path.exists(): + new_path.rename(old_path) + except OSError as exc: + self.log.warning(f"Could not undo rename of {dest}: {exc}") + self._report_error(new_path, f"rename reverted; database move failed (pk {pk})") + + def _rename_kept_conversions( + self, + task: BulkTagWriteTask, + current_paths: dict[int, Path], + written_paths: dict[int, Path], + converted_paths: dict[int, Path], + ) -> dict[int, Path]: + """Give the new CBZ of a kept-original conversion its scheme name.""" + if not task.rename or task.delete_original: + return {} + config = build_predict_config(task.delete_keys, task.mode) + post_renamed: dict[int, Path] = {} + for pk, written_path in written_paths.items(): + if pk in converted_paths or written_path == current_paths[pk]: + continue try: - new_path = self._rename_one(old_path) + name = predict_name(written_path, None, config) + if not name: + continue + target = written_path.parent / name + if target == written_path or target.exists(): + continue + written_path.rename(target) except Exception as exc: - self.log.warning(f"Rename error for {old_path}: {exc}") - add_tag_write_error(str(old_path), f"rename failed: {exc}") - had_errors = True + self._report_error(written_path, f"rename failed: {exc}") continue - if new_path is None or new_path == old_path: + post_renamed[pk] = target + return post_renamed + + def _report_error(self, path: Path, reason: str) -> None: + """Surface a per-file failure to admins (badge + Tagging panel).""" + self.log.warning(f"Tag write: {reason} for {path}") + add_tag_write_error(str(path), reason) + self.librarian_queue.put(TAG_WRITE_ERRORS_CHANGED_TASK) + + def _apply_moves_inline( + self, library_id: int, moves: dict[int, tuple[str, str]] + ) -> set[int]: + """ + Apply path moves to the database now, and report which didn't take. + + Runs the importer's own move phase rather than a hand-rolled path + update: it drops moves onto occupied destinations, keeps the stored + stat a rename doesn't change, and re-parents the row. Constructing + the importer costs one query. Its ``apply``/``finish`` are never + called — ``finish`` would end this batch's live progress status. + + Failures are read back from the database rather than inferred from + the move phase's count, which reports nothing about *which* comic + it dropped and can come back zero after the rows were already + updated. + """ + if not moves: + return set() + start_time = now() + import_task = ImportTask( + library_id=library_id, + files_moved=dict(moves.values()), + ) + importer = ComicImporter( + import_task, + self.log, + self.librarian_queue, + self.db_write_lock, + self.abort_event, + ) + try: + importer.bulk_comics_moved() + except Exception: + self.log.exception(f"Applying tag write moves in library {library_id}") + landed = dict(Comic.objects.filter(pk__in=moves).values_list("pk", "path")) + failed = {pk for pk, (_, dest) in moves.items() if landed.get(pk) != dest} + TimestampUpdater( + self.log, self.librarian_queue, self.db_write_lock + ).update_library_collections(importer.library, start_time, {}) + # The browser's page-mtime cache would otherwise serve the old + # paths for its TTL. + cache.clear() + self.librarian_queue.put(LIBRARY_CHANGED_TASK) + return failed + + def _sync_conversions( + self, + task: BulkTagWriteTask, + current_paths: dict[int, Path], + written_paths: dict[int, Path], + lib_of: dict[int, int], + ) -> dict[int, Path]: + """ + Move rows onto the CBZ a conversion produced, and report which moved. + + A repacked archive is a new file at a new path, and the original is + gone under ``delete_original`` — neither scanner can pair that into + a move, so codex has to state it. With the original kept, the row + stays on it and the CBZ is a separate new file. + """ + if not task.delete_original: + return {} + moves: defaultdict[int, dict[int, tuple[str, str]]] = defaultdict(dict) + converted: dict[int, Path] = {} + for pk, written_path in written_paths.items(): + source = current_paths[pk] + if written_path == source: continue - moved[lib_of[pk]][str(old_path)] = str(new_path) - renamed_pks.add(pk) - if had_errors: - self.librarian_queue.put(TAG_WRITE_ERRORS_CHANGED_TASK) - self._enqueue_rename_imports(moved, library_events, tags_written=tags_written) - return renamed_pks + moves[lib_of[pk]][pk] = (str(source), str(written_path)) + converted[pk] = written_path + for library_id, library_moves in moves.items(): + for pk in self._apply_moves_inline(library_id, library_moves): + self.log.warning( + f"Tag write: conversion move failed for {library_moves[pk][1]}" + ) + converted.pop(pk, None) + return converted - def _enqueue_rename_imports( + def _enqueue_rereads( self, - moved: dict[int, dict[str, str]], - library_events: dict[int, bool], - *, - tags_written: bool, + current_paths: dict[int, Path], + written_paths: dict[int, Path], + converted_paths: dict[int, Path], + post_renamed: dict[int, Path], + lib_of: dict[int, int], ) -> None: """ - Sync the DB for renamed comics, watcher-aware. - - Watched libraries: enqueue nothing — the watcher's inode move-detection - emits the ``files_moved`` import, and ``build_import_task`` remaps the - tag-write's modify event from the pre-rename path onto the move - destination, so the new tags re-read from the same batch. A - self-enqueued move would only duplicate it. - Unwatched libraries: enqueue one targeted move import that updates the - path and, when tags were written, re-reads the new file's metadata - (``move_and_modify_dirs`` runs before the per-comic ``read`` phase). + Ask the importer to re-read the metadata codex just wrote. + + Every move is already applied, so these tasks only name paths the + database holds: a scan that beats them to the queue reconciles + against rows that are already correct. Watched libraries would + eventually re-read from the watcher's own modify event, but only + with the import-metadata flag on, so the re-read is requested + either way. """ - for library_id, lib_moved in moved.items(): - if library_events.get(library_id): - continue - files_modified = ( - frozenset(lib_moved.values()) if tags_written else frozenset() - ) + modified: defaultdict[int, set[str]] = defaultdict(set) + created: defaultdict[int, set[str]] = defaultdict(set) + for pk, written_path in written_paths.items(): + library_id = lib_of[pk] + if pk in converted_paths: + modified[library_id].add(str(converted_paths[pk])) + elif written_path != current_paths[pk]: + # A kept original's new CBZ is a file the database has + # never seen. + created[library_id].add(str(post_renamed.get(pk, written_path))) + else: + modified[library_id].add(str(written_path)) + # A rename with no write changed no metadata, so nothing to re-read. + for library_id in modified.keys() | created.keys(): import_task = ImportTask( library_id=library_id, - files_moved=dict(lib_moved), - files_modified=files_modified, + files_modified=frozenset(modified.get(library_id, ())), + files_created=frozenset(created.get(library_id, ())), force_import_metadata=True, check_metadata_mtime=False, ) diff --git a/codex/librarian/scribe/tagwrite_rename.py b/codex/librarian/scribe/tagwrite_rename.py new file mode 100644 index 000000000..3d1c37350 --- /dev/null +++ b/codex/librarian/scribe/tagwrite_rename.py @@ -0,0 +1,97 @@ +""" +Predict where a tag write will leave each archive. + +Renaming happens *before* the write, so the destination has to be known +in advance rather than read back off the finished file. Comicbox does the +predicting — the name it renders is exactly what ``rename_file`` would +use — but it has to be handed the same settings and pending patch the +write will apply, or the rename lands somewhere the write never agreed +to. + +The admin preflight preview derives its names through here too, so the +dialog cannot promise a name the rename won't produce. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING + +from comicbox.box import Comicbox +from comicbox.config.settings import WriteMode + +from codex.settings import COMICBOX_CONFIG + +if TYPE_CHECKING: + from collections.abc import Collection, Mapping + from pathlib import Path + +#: Archives comicbox writes in place. Every other format is repacked as a +#: CBZ at a new path, which is what makes a conversion a *move* rather +#: than a modification. +_WRITE_IN_PLACE_SUFFIXES = frozenset({".cbz", ".pdf"}) +_CBZ_SUFFIX = ".cbz" + + +@dataclass(frozen=True, slots=True) +class RenamePlan: + """Where one comic's archive is headed.""" + + pk: int + old_path: Path + #: The pre-write rename destination. Keeps the archive's current + #: suffix: a CBR is still a CBR until the write repacks it. + target: Path + #: Where the file ends up once the write's conversion (if any) is + #: done. Only differs from ``target`` for a converting archive. + final_path: Path + + +def build_predict_config(delete_keys: Collection[str] | None, mode: str): + """ + Return the config a write with these settings would parse under. + + Mirrors comicbox's own ``_build_write_settings``: the write mode + picks the merger that decides whether a patch value replaces or + extends what the archive already holds, and cleared fields must + vanish from the predicted name exactly as the write will clear them. + """ + write = replace(COMICBOX_CONFIG.write, mode=WriteMode(mode)) + keys = frozenset( + key.removeprefix("comicbox.") for key in (delete_keys or ()) if key + ) + if not keys: + return replace(COMICBOX_CONFIG, write=write) + general = replace( + COMICBOX_CONFIG.general, + delete_keys=COMICBOX_CONFIG.general.delete_keys | keys, + ) + return replace(COMICBOX_CONFIG, write=write, general=general) + + +def predict_name(path: Path, patch: Mapping | None, config) -> str: + """ + Return the scheme name a write carrying ``patch`` would rename to. + + Empty when no usable name could be built. Opens the archive. + """ + metadata = {"comicbox": dict(patch)} if patch else None + with Comicbox(path, config=config, metadata=metadata) as car: + return car.predict_filename() + + +def will_convert(path: Path) -> bool: + """Whether writing this archive repacks it as a CBZ at a new path.""" + return path.suffix.lower() not in _WRITE_IN_PLACE_SUFFIXES + + +def plan_rename( + pk: int, old_path: Path, patch: Mapping | None, config +) -> RenamePlan | None: + """Build one comic's rename plan, or None when there is no name.""" + name = predict_name(old_path, patch, config) + if not name: + return None + target = old_path.parent / name + final_path = target.with_suffix(_CBZ_SUFFIX) if will_convert(target) else target + return RenamePlan(pk=pk, old_path=old_path, target=target, final_path=final_path) diff --git a/codex/views/admin/onlinetag.py b/codex/views/admin/onlinetag.py index 06989ca94..1dc838c5b 100644 --- a/codex/views/admin/onlinetag.py +++ b/codex/views/admin/onlinetag.py @@ -68,6 +68,20 @@ def get(self, _request): return Response({"session_id": sid}) +def _review_sources_by_pk() -> dict[int, tuple[str, ...]]: + """Map each comic still awaiting review to the source(s) that prompted.""" + by_pk: dict[int, list[str]] = {} + for prompt in get_pending_prompts().values(): + pk = prompt.get("pk") + if pk is None: + continue + sources = by_pk.setdefault(pk, []) + source = prompt.get("source") + if source and source not in sources: + sources.append(source) + return {pk: tuple(sources) for pk, sources in by_pk.items()} + + class AdminOnlineTagSnapshotView(AdminAPIView): """Return the live (or last-finished) online tagging session snapshot.""" @@ -75,10 +89,15 @@ def get(self, _request): """Return the session snapshot, reconciled with current resolutions.""" snapshot = get_snapshot() if snapshot: - review_pks = {p.get("pk") for p in get_pending_prompts().values()} snapshot = overlay_resolutions( - snapshot, review_pks, get_resolved_outcomes() + snapshot, _review_sources_by_pk(), get_resolved_outcomes() ) + # The frozen snapshot's own flag only reports that comics were + # left unprocessed; Resume needs the descriptor that says which + # ones and how. They can diverge (a daemon killed mid-scan), and + # offering a button that 400s is worse than not offering it. + resume = get_resume_state() + snapshot["resumable"] = bool(resume and resume.get("remaining_pks")) return Response({"snapshot": snapshot}) diff --git a/codex/views/admin/tagwrite.py b/codex/views/admin/tagwrite.py index 9c0aa85ad..a75228b51 100644 --- a/codex/views/admin/tagwrite.py +++ b/codex/views/admin/tagwrite.py @@ -2,23 +2,20 @@ import json from collections.abc import Sequence -from dataclasses import replace from pathlib import Path from types import MappingProxyType from typing import override -from comicbox.box import Comicbox -from comicbox.formats import MetadataFormats from rest_framework.permissions import BasePermission, IsAdminUser from rest_framework.response import Response from rest_framework.status import HTTP_202_ACCEPTED from codex.librarian.mp_queue import LIBRARIAN_QUEUE +from codex.librarian.scribe.tagwrite_rename import build_predict_config, plan_rename from codex.librarian.scribe.tasks import BulkTagWriteTask from codex.models.admin import ComicboxTaggingDefaults from codex.models.comic import Comic from codex.serializers.admin.tagging import TagWriteRequestSerializer -from codex.settings import COMICBOX_CONFIG from codex.views.admin.auth import AdminAPIView from codex.views.admin.identifier_parse import parse_identifier_url from codex.views.browser.filters.filter import BrowserFilterView @@ -104,41 +101,35 @@ class AdminTagWritePreflightView(FilteredComicPksView): """Check how many comics need conversion before writing.""" @staticmethod - def _preview_one(old_path: Path, metadata: dict | None, config) -> str: + def _preview_one(pk: int, old_path: Path, patch: dict | None, config) -> str: """ - Return the comicbox-scheme name the given patch produces for one comic. + Return the name this comic ends up with, or "" if it can't be built. - Overlays the pending (unsaved) patch onto the archive's metadata in - memory and serializes the FILENAME format — the same construction - ``rename_file`` uses — so the dialog can show the would-be name. Opens - the archive (I/O). Returns "" when no name could be built. + Derived through the same planner the rename itself uses, so the + dialog cannot promise a name the write won't produce. Shows the + *final* name, which for an archive the write will repack is the + converted CBZ rather than the interim one. Opens the archive (I/O). """ try: - with Comicbox(old_path, config=config, metadata=metadata) as car: - return car.to_string(MetadataFormats.FILENAME) or "" + plan = plan_rename(pk, old_path, patch, config) except Exception: return "" + return plan.final_path.name if plan else "" def _filename_previews( self, comic_pks: frozenset[int], patch_str: str, delete_keys: tuple[str, ...] = (), + mode: str = "additive", ) -> list[dict[str, str]]: """Preview the rename (old → new) for each selected comic, capped.""" patch = json.loads(patch_str or "null") - metadata = {"comicbox": patch} if patch else None # Pending cleared fields must vanish from the previewed name exactly - # as the real write (BulkWriteItem.delete_keys) will clear them. - config = COMICBOX_CONFIG - if delete_keys: - config = replace( - config, - general=replace( - config.general, - delete_keys=config.general.delete_keys | frozenset(delete_keys), - ), - ) + # as the real write (BulkWriteItem.delete_keys) will clear them, and + # the write mode decides whether a patch value replaces or extends + # what the archive already holds. + config = build_predict_config(delete_keys, mode) comics = ( Comic.objects.filter(pk__in=comic_pks) .only("pk", "path") @@ -150,7 +141,7 @@ def _filename_previews( previews.append( { "old": old_path.name, - "new": self._preview_one(old_path, metadata, config), + "new": self._preview_one(comic.pk, old_path, patch, config), } ) return previews @@ -190,6 +181,7 @@ def post(self, request): comic_pks, data.get("patch") or "", tuple(data.get("delete_keys") or ()), + data["mode"], ), "skipped": self.skipped_read_only, } diff --git a/frontend/bun.lock b/frontend/bun.lock index 6659bd21a..3a84ed9f1 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -1,5 +1,5 @@ { - "lockfileVersion": 1, + "lockfileVersion": 2, "configVersion": 1, "workspaces": { "": { @@ -13,7 +13,7 @@ "pretty-bytes": "^7.1.1", "text-case": "^1.2.11", "vue": "^3.5.41", - "vue-pdf-embed": "^2.1.5", + "vue-pdf-embed": "^2.1.6", "vue-router": "^5.2.0", "vuetify": "^4.1.11", "xior": "^0.8.4", @@ -22,7 +22,7 @@ "@mdi/font": "^7.4.47", "@mdi/js": "^7.4.47", "@pinia/testing": "^2.0.1", - "@types/node": "^26.2.0", + "@types/node": "^26.3.0", "@unhead/bundler": "^3.4.0", "@vitejs/plugin-vue": "^6.0.8", "@vue/test-utils": "^2.4.11", @@ -104,29 +104,29 @@ "@mdi/js": ["@mdi/js@7.4.47", "", {}, "sha512-KPnNOtm5i2pMabqZxpUz7iQf+mfrYZyKCZ8QNz85czgEt7cuHcGorWfdzUMWYA0SD+a6Hn4FmJ+YhzzzjkTZrQ=="], - "@napi-rs/canvas": ["@napi-rs/canvas@0.1.100", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.100", "@napi-rs/canvas-darwin-arm64": "0.1.100", "@napi-rs/canvas-darwin-x64": "0.1.100", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", "@napi-rs/canvas-linux-arm64-musl": "0.1.100", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", "@napi-rs/canvas-linux-x64-gnu": "0.1.100", "@napi-rs/canvas-linux-x64-musl": "0.1.100", "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", "@napi-rs/canvas-win32-x64-msvc": "0.1.100" } }, "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA=="], + "@napi-rs/canvas": ["@napi-rs/canvas@1.0.8", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.8", "@napi-rs/canvas-darwin-arm64": "1.0.8", "@napi-rs/canvas-darwin-x64": "1.0.8", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.8", "@napi-rs/canvas-linux-arm64-gnu": "1.0.8", "@napi-rs/canvas-linux-arm64-musl": "1.0.8", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.8", "@napi-rs/canvas-linux-x64-gnu": "1.0.8", "@napi-rs/canvas-linux-x64-musl": "1.0.8", "@napi-rs/canvas-win32-arm64-msvc": "1.0.8", "@napi-rs/canvas-win32-x64-msvc": "1.0.8" } }, "sha512-/SaLcvlqGWdm0HSCWMgHu7cjJiQXfP8/mOY+6dUyV9flQz7sPBBZ+ed2zYtoukojPmxOaL7bm+d/G4GeWWoN7g=="], - "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.100", "", { "os": "android", "cpu": "arm64" }, "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ=="], + "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.8", "", { "os": "android", "cpu": "arm64" }, "sha512-5+nkh8i3gt6lqS/d2jTZ1xAn6tdgtB4Lf1mW6T0Qm5/rXNwBuV1sAEyLEWan5o9gJPU/GuvHR3rvSeZ+FaGrbw=="], - "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@0.1.100", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A=="], + "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.8", "", { "os": "darwin", "cpu": "arm64" }, "sha512-7jQ47gi+fZ7KJmfc/5rNyy1CYw/cu4kZ0KPIYbo9UUgSdW0bKQJpt+WihEor6s4Lyp7+xc3a+3HeyXmAEbbnPg=="], - "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@0.1.100", "", { "os": "darwin", "cpu": "x64" }, "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw=="], + "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.8", "", { "os": "darwin", "cpu": "x64" }, "sha512-rRjDMZs9pIRKGxgijwezplKc1RnJsqUokrA9h88bbTkqQ+7ePj0ZN4ZnZDy8Vu0tXs7KRlI2tQLaK4mx9QlxHg=="], - "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@0.1.100", "", { "os": "linux", "cpu": "arm" }, "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA=="], + "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.8", "", { "os": "linux", "cpu": "arm" }, "sha512-jGcCd+8ra6Q61xKqZeiItujTpp9a9eRLcQ0jW6qYNku+WpupqOPFPY0SrsuSnXFviJwkpKYT9p7QrB4lsf3LNQ=="], - "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@0.1.100", "", { "os": "linux", "cpu": "arm64" }, "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw=="], + "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-od6I2Y7kU7i1SwZYG2EKW8rWz6JiedtPpko4WEe1DDsiikrfaotVBCRaUTM5/yeZKaZ92EatoAS+5xG+6uJlYA=="], - "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@0.1.100", "", { "os": "linux", "cpu": "arm64" }, "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ=="], + "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-yYkPbJDJiWj6N0gASA3CAvRypZmVpJnxU0DQg3aBhneLDQde9TPLKADsQkobNoJUtTT/lj46aWpzT48PDb3Qcg=="], - "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@0.1.100", "", { "os": "linux", "cpu": "none" }, "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw=="], + "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.8", "", { "os": "linux", "cpu": "none" }, "sha512-PB00MSKAp4VwK/xwe6duKxRKmH8UH4GIl1pqHSbxng0jnU9Dr7FwaDypDiqwNFZ774N+8G7mJLGuLtg9NTcQsg=="], - "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@0.1.100", "", { "os": "linux", "cpu": "x64" }, "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg=="], + "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.8", "", { "os": "linux", "cpu": "x64" }, "sha512-TWM2XWJoitLiIPCvgJh7SriC+L/T9qkYCVzC66AidsZy0QP1hkKzBzVwshCdcA3q6fIn3yE0ISbq4lMJSy8jFw=="], - "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@0.1.100", "", { "os": "linux", "cpu": "x64" }, "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA=="], + "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.8", "", { "os": "linux", "cpu": "x64" }, "sha512-hb20MxKXXb5IB7AAwN8UHz9WRsa2HmdZfjsDCzjElwJoeV1aotVEwFU4FrFQcYQVzsJQLeaCc/2Qdt/0Q72mMg=="], - "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@0.1.100", "", { "os": "win32", "cpu": "arm64" }, "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw=="], + "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.8", "", { "os": "win32", "cpu": "arm64" }, "sha512-WwPN08IXE4SkL+FhJyPz/iFnycMAUkbphFIT4cmKLlvbSU0Zfn1R7BGJ3Hqky1S89QUYc0Q4IOScXb/42Re9wQ=="], - "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@0.1.100", "", { "os": "win32", "cpu": "x64" }, "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA=="], + "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.8", "", { "os": "win32", "cpu": "x64" }, "sha512-XkrVqKb+pxyba7kjy2LJvABFVBTE0DNpEl7MrG4OYUmaWarrXH+t54z/Czj2YxCKtizYTV4mg6phNm3x24qjhQ=="], "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.2.3", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" } }, "sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q=="], @@ -280,7 +280,7 @@ "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], - "@types/node": ["@types/node@26.2.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg=="], + "@types/node": ["@types/node@26.3.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-L3fgrnchriRC2ExBflb8j4uZZURHZfQsmQeyVzhjcHW4kkwVyo8/0h1B2MVzMTrYUJYu6G7EWs14hW/L9putqw=="], "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], @@ -450,7 +450,7 @@ "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], - "eslint": ["eslint@10.9.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.5", "@eslint/config-helpers": "^0.7.0", "@eslint/core": "^1.2.1", "@eslint/plugin-kit": "^0.7.2", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "minimatch": "^10.2.5", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "peerDependencies": { "jiti": "*" }, "optionalPeers": ["jiti"], "bin": { "eslint": "bin/eslint.js" } }, "sha512-5KeEOJZBfEVA47boFiBsf+6MmmJpffM7qEBg4pLla2e4nlKgdKlqCW0oSLOGsT8Wl5uCGJptLV1bkaiShj90Gw=="], + "eslint": ["eslint@10.9.1", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.5", "@eslint/config-helpers": "^0.7.0", "@eslint/core": "^1.2.1", "@eslint/plugin-kit": "^0.7.2", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "minimatch": "^10.2.5", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "peerDependencies": { "jiti": "*" }, "optionalPeers": ["jiti"], "bin": { "eslint": "bin/eslint.js" } }, "sha512-9VaAkDURekixUQJy0oJYl2DcN6oKMfxay7XzaGYAWQwsb6qfKf+x76R2k1L8kb1boc+FyCAaTA9GmiKaaiaF+A=="], "eslint-import-resolver-oxc": ["eslint-import-resolver-oxc@0.15.0", "", { "dependencies": { "stable-hash-x": "^0.1.1" }, "peerDependencies": { "@rspack/cli": "*", "@rspack/core": "*", "eslint": "*", "eslint-plugin-import": "*", "eslint-plugin-import-x": "*", "oxc-resolver": "*", "vite": "*", "webpack": "*", "webpack-cli": "*", "webpack-merge": "*" }, "optionalPeers": ["@rspack/cli", "@rspack/core", "eslint-plugin-import", "eslint-plugin-import-x", "vite", "webpack", "webpack-cli", "webpack-merge"] }, "sha512-qod5T3t3PbJdMZX3GYb8rdhD/UBK9rJCSE2b2+ttXtVdBm39CHgyCht1pMXj/1Z0ko1wNme+az4Z5chowgXD5Q=="], @@ -642,13 +642,13 @@ "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - "pdfjs-dist": ["pdfjs-dist@5.7.284", "", { "optionalDependencies": { "@napi-rs/canvas": "^0.1.100" } }, "sha512-h4EdYQczmGhbOlqc3PPZwxevn7ApdWPbovAuWXOB/DjIyigSnwfy2oze7c6mRcSr9XgLp3eN3EeL4DyySTPMFw=="], + "pdfjs-dist": ["pdfjs-dist@6.2.108", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.0" } }, "sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ=="], "perfect-debounce": ["perfect-debounce@2.1.0", "", {}, "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g=="], "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], + "picomatch": ["picomatch@4.0.7", "", {}, "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA=="], "pinia": ["pinia@4.0.3", "", { "dependencies": { "nostics": "^1.1.4" }, "peerDependencies": { "@vue/devtools-api": "^8.1.5", "typescript": ">=5.6.0", "vue": "^3.5.11" }, "optionalPeers": ["typescript"] }, "sha512-XMQqpvjgG7LMqVhhFUzKLT4KEbsYbfZZ0CZU9PgdFm1O2VKBmIkykL8+SfNhOaT7sR0wT6BEgKT0YNDYWgmVRA=="], @@ -804,7 +804,7 @@ "vitest": ["vitest@4.1.11", "", { "dependencies": { "@vitest/expect": "4.1.11", "@vitest/mocker": "4.1.11", "@vitest/pretty-format": "4.1.11", "@vitest/runner": "4.1.11", "@vitest/snapshot": "4.1.11", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.11", "@vitest/browser-preview": "4.1.11", "@vitest/browser-webdriverio": "4.1.11", "@vitest/coverage-istanbul": "4.1.11", "@vitest/coverage-v8": "4.1.11", "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw=="], - "vscode-uri": ["vscode-uri@3.1.0", "", {}, "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ=="], + "vscode-uri": ["vscode-uri@3.2.0", "", {}, "sha512-m2gXo3bn0G1kT9InzMf07fTbqMbGtyckj3bH5ktLO+1Ssv+yiATZ4dhwaQv9UZWxJh6E9IFGnQyjgWVDWVBDrg=="], "vue": ["vue@3.5.41", "", { "dependencies": { "@vue/compiler-dom": "3.5.41", "@vue/compiler-sfc": "3.5.41", "@vue/runtime-dom": "3.5.41", "@vue/server-renderer": "3.5.41", "@vue/shared": "3.5.41" }, "peerDependencies": { "typescript": "*" }, "optionalPeers": ["typescript"] }, "sha512-2laE0p+aK+/AOPG/XL/WepOs/GlK755LJ1XECi9kDUrz1FKNw8rb2Xzlw9JS1rqEV55nb0ttsKxVlTCcd+R5cg=="], @@ -814,7 +814,7 @@ "vue-eslint-parser": ["vue-eslint-parser@10.4.1", "", { "dependencies": { "debug": "^4.4.0", "eslint-scope": "^8.2.0 || ^9.0.0", "eslint-visitor-keys": "^4.2.0 || ^5.0.0", "espree": "^10.3.0 || ^11.0.0", "esquery": "^1.6.0", "semver": "^7.6.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0" } }, "sha512-Gk6gRDj0n/fkRa3C3l0bBheoBckUq/Rs0F/TvMWIS6nzzx67amAViMe9CkNgsP2tXyQONvGiHQESHwFtZ3aYDA=="], - "vue-pdf-embed": ["vue-pdf-embed@2.1.5", "", { "dependencies": { "pdfjs-dist": "^5.7.284" }, "peerDependencies": { "vue": "^3.3.0" } }, "sha512-IGFVBYlnOz2zSql1zk4YJyBu584EZa6RUykk5f8wkHF/AR31khCa+ruJoRag+Ff2UyntkWu0brENIKoikQ7F8g=="], + "vue-pdf-embed": ["vue-pdf-embed@2.1.6", "", { "dependencies": { "pdfjs-dist": "^6.2.108" }, "peerDependencies": { "vue": "^3.3.0" } }, "sha512-cNmclL9jGuRIqgm13fx/3ZZd1nW/Nn0zXfQUo/yd8AQU9GTFLLYRgFc5HM41v1+bMoj1wW59mZoB/cNQx8Y2Rw=="], "vue-router": ["vue-router@5.2.0", "", { "dependencies": { "@babel/generator": "^8.0.0", "@vue-macros/common": "^3.1.3", "@vue/devtools-api": "^8.1.5", "ast-walker-scope": "^0.9.0", "chokidar": "^5.0.0", "json5": "^2.2.3", "local-pkg": "^1.2.1", "magic-string": "^0.30.21", "mlly": "^1.8.2", "muggle-string": "^0.4.1", "nostics": "^1.1.4", "pathe": "^2.0.3", "picomatch": "^4.0.5", "scule": "^1.3.0", "tinyglobby": "^0.2.17", "unplugin": "^3.3.0", "unplugin-utils": "^0.3.2", "yaml": "^2.9.0" }, "peerDependencies": { "@pinia/colada": ">=0.21.2", "@vue/compiler-sfc": "^3.5.34 || ^4.0.0", "pinia": "^3.0.4 || ^4.0.2", "vite": "^7.3.0 || ^8.0.0", "vue": "^3.5.34 || ^4.0.0" }, "optionalPeers": ["@pinia/colada", "@vue/compiler-sfc", "pinia", "vite"] }, "sha512-QAC5i0LEb1GLG0LXDQmHu8L7FX12j0KwU/JTKmLQUJMrn04gQdKP6Du+p0QwpHb3iy71vBlqnHQ8WAfOSAWhqw=="], diff --git a/frontend/package.json b/frontend/package.json index 31f2deb92..813e5b3d1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "codex", - "version": "2.2.10", + "version": "2.2.11", "private": true, "description": "ui for codex api", "type": "module", @@ -24,7 +24,7 @@ "pretty-bytes": "^7.1.1", "text-case": "^1.2.11", "vue": "^3.5.41", - "vue-pdf-embed": "^2.1.5", + "vue-pdf-embed": "^2.1.6", "vue-router": "^5.2.0", "vuetify": "^4.1.11", "xior": "^0.8.4" @@ -33,7 +33,7 @@ "@mdi/font": "^7.4.47", "@mdi/js": "^7.4.47", "@pinia/testing": "^2.0.1", - "@types/node": "^26.2.0", + "@types/node": "^26.3.0", "@unhead/bundler": "^3.4.0", "@vitejs/plugin-vue": "^6.0.8", "@vue/test-utils": "^2.4.11", diff --git a/frontend/src/components/admin/tabs/tagging-status-table.vue b/frontend/src/components/admin/tabs/tagging-status-table.vue index 9a3c39f3e..2dba09bef 100644 --- a/frontend/src/components/admin/tabs/tagging-status-table.vue +++ b/frontend/src/components/admin/tabs/tagging-status-table.vue @@ -129,10 +129,24 @@ height="420" density="compact" > -