From a243a8f8c1a048eebb1ce8ae4bc29825e2aa2e78 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:04:21 +0300 Subject: [PATCH 01/12] feat(payload): add import-export plugin for promises and ai-extractions Enable data import/export functionality for the "promises" and "ai-extractions" collections via the Payload CMS import-export plugin. This provides administrators with CSV/JSON export capabilities and data import functionality directly from the admin interface. --- package.json | 1 + pnpm-lock.yaml | 69 +++++++++ src/app/(payload)/admin/importMap.js | 26 ++++ src/payload-types.ts | 200 ++++++++++++++++++++++++++- src/plugins/index.ts | 12 ++ 5 files changed, 307 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 54d84893..4d36bcdc 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "@next/third-parties": "^15.5.3", "@payloadcms/db-mongodb": "3.77.0", "@payloadcms/email-nodemailer": "3.77.0", + "@payloadcms/plugin-import-export": "3.77.0", "@payloadcms/next": "3.77.0", "@payloadcms/payload-cloud": "3.77.0", "@payloadcms/plugin-cloud-storage": "3.77.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fd38ed9..47455898 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -95,6 +95,9 @@ importers: '@payloadcms/plugin-cloud-storage': specifier: 3.77.0 version: 3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2) + '@payloadcms/plugin-import-export': + specifier: 3.77.0 + version: 3.77.0(@payloadcms/ui@3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3) '@payloadcms/plugin-multi-tenant': specifier: 3.77.0 version: 3.77.0(@payloadcms/ui@3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2)) @@ -1720,78 +1723,92 @@ packages: resolution: {integrity: sha512-RXwd0CgG+uPRX5YYrkzKyalt2OJYRiJQ8ED/fi1tq9WQW2jsQIn0tqrlR5l5dr/rjqq6AHAxURhj2DVjyQWSOA==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.0': resolution: {integrity: sha512-mWd2uWvDtL/nvIzThLq3fr2nnGfyr/XMXlq8ZJ9WMR6PXijHlC3ksp0IpuhK6bougvQrchUAfzRLnbsen0Cqvw==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.0': resolution: {integrity: sha512-Xod/7KaDDHkYu2phxxfeEPXfVXFKx70EAFZ0qyUdOjCcxbjqyJOEUpDe6RIyaunGxT34Anf9ue/wuWOqBW2WcQ==} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.0': resolution: {integrity: sha512-eMKfzDxLGT8mnmPJTNMcjfO33fLiTDsrMlUVcp6b96ETbnJmd4uvZxVJSKPQfS+odwfVaGifhsB07J1LynFehw==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.0': resolution: {integrity: sha512-ZW3FPWIc7K1sH9E3nxIGB3y3dZkpJlMnkk7z5tu1nSkBoCgw2nSRTFHI5pB/3CQaJM0pdzMF3paf9ckKMSE9Tg==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.0': resolution: {integrity: sha512-UG+LqQJbf5VJ8NWJ5Z3tdIe/HXjuIdo4JeVNADXBFuG7z9zjoegpzzGIyV5zQKi4zaJjnAd2+g2nna8TZvuW9Q==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.0': resolution: {integrity: sha512-SRYOLR7CXPgNze8akZwjoGBoN1ThNZoqpOgfnOxmWsklTGVfJiGJoC/Lod7aNMGA1jSsKWM1+HRX43OP6p9+6Q==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.34.3': resolution: {integrity: sha512-QdrKe3EvQrqwkDrtuTIjI0bu6YEJHTgEeqdzI3uWJOH6G1O8Nl1iEeVYRGdj1h5I21CqxSvQp1Yv7xeU3ZewbA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.34.3': resolution: {integrity: sha512-oBK9l+h6KBN0i3dC8rYntLiVfW8D8wH+NPNT3O/WBHeW0OQWCjfWksLUaPidsrDKpJgXp3G3/hkmhptAW0I3+A==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-ppc64@0.34.3': resolution: {integrity: sha512-GLtbLQMCNC5nxuImPR2+RgrviwKwVql28FWZIW1zWruy6zLgA5/x2ZXk3mxj58X/tszVF69KK0Is83V8YgWhLA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.34.3': resolution: {integrity: sha512-3gahT+A6c4cdc2edhsLHmIOXMb17ltffJlxR0aC2VPZfwKoTGZec6u5GrFgdR7ciJSsHT27BD3TIuGcuRT0KmQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.34.3': resolution: {integrity: sha512-8kYso8d806ypnSq3/Ly0QEw90V5ZoHh10yH0HnrzOCr6DKAPI6QVHvwleqMkVQ0m+fc7EH8ah0BB0QPuWY6zJQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.3': resolution: {integrity: sha512-vAjbHDlr4izEiXM1OTggpCcPg9tn4YriK5vAjowJsHwdBIdx0fYRsURkxLG2RLm9gyBq66gwtWI8Gx0/ov+JKQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.34.3': resolution: {integrity: sha512-gCWUn9547K5bwvOn9l5XGAEjVTTRji4aPTqLzGXHvIr6bIDZKNTA34seMPgM0WmSf+RYBH411VavCejp3PkOeQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.34.3': resolution: {integrity: sha512-+CyRcpagHMGteySaWos8IbnXcHgfDn7pO2fiC2slJxvNq9gDipYBN42/RagzctVRKgxATmfqOSulgZv5e1RdMg==} @@ -2073,24 +2090,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@next/swc-linux-arm64-musl@15.5.14': resolution: {integrity: sha512-8B8cngBaLadl5lbDRdxGCP1Lef8ipD6KlxS3v0ElDAGil6lafrAM3B258p1KJOglInCVFUjk751IXMr2ixeQOQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@next/swc-linux-x64-gnu@15.5.14': resolution: {integrity: sha512-bAS6tIAg8u4Gn3Nz7fCPpSoKAexEt2d5vn1mzokcqdqyov6ZJ6gu6GdF9l8ORFrBuRHgv3go/RfzYz5BkZ6YSQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@next/swc-linux-x64-musl@15.5.14': resolution: {integrity: sha512-mMxv/FcrT7Gfaq4tsR22l17oKWXZmH/lVqcvjX0kfp5I0lKodHYLICKPoX1KRnnE+ci6oIUdriUhuA3rBCDiSw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@next/swc-win32-arm64-msvc@15.5.14': resolution: {integrity: sha512-OTmiBlYThppnvnsqx0rBqjDRemlmIeZ8/o4zI7veaXoeO1PVHoyj2lfTfXTiiGjCyRDhA10y4h6ZvZvBiynr2g==} @@ -2564,6 +2585,12 @@ packages: react: ^19.0.1 || ^19.1.2 || ^19.2.1 react-dom: ^19.0.1 || ^19.1.2 || ^19.2.1 + '@payloadcms/plugin-import-export@3.77.0': + resolution: {integrity: sha512-6E3NJkcWER10h7H7ViCja55fijmvrimzK1AG9Fc1oMpAr0mEe9MGaj7KRMa9eNLvGrS343TdFW0CkBV9JB05qw==} + peerDependencies: + '@payloadcms/ui': 3.77.0 + payload: 3.77.0 + '@payloadcms/plugin-multi-tenant@3.77.0': resolution: {integrity: sha512-EQuG/O3xsrN5yyGq2AfErrK/1KKkTXKnBzMq2l60Yl1xVAw5FkjVa+SC+pJJtrvNSBOHQxTy6vsHANPghiu7kw==} peerDependencies: @@ -2690,56 +2717,67 @@ packages: resolution: {integrity: sha512-9fhTJyOb275w5RofPSl8lpr4jFowd+H4oQKJ9XTYzD1JWgxdZKE8bA6d4npuiMemkecQOcigX01FNZNCYnQBdA==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.46.4': resolution: {integrity: sha512-+6kCIM5Zjvz2HwPl/udgVs07tPMIp1VU2Y0c72ezjOvSvEfAIWsUgpcSDvnC7g9NrjYR6X9bZT92mZZ90TfvXw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.46.4': resolution: {integrity: sha512-SWuXdnsayCZL4lXoo6jn0yyAj7TTjWE4NwDVt9s7cmu6poMhtiras5c8h6Ih6Y0Zk6Z+8t/mLumvpdSPTWub2Q==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.46.4': resolution: {integrity: sha512-vDknMDqtMhrrroa5kyX6tuC0aRZZlQ+ipDfbXd2YGz5HeV2t8HOl/FDAd2ynhs7Ki5VooWiiZcCtxiZ4IjqZwQ==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.46.4': resolution: {integrity: sha512-mCBkjRZWhvjtl/x+Bd4fQkWZT8canStKDxGrHlBiTnZmJnWygGcvBylzLVCZXka4dco5ymkWhZlLwKCGFF4ivw==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-gnu@4.46.4': resolution: {integrity: sha512-YMdz2phOTFF+Z66dQfGf0gmeDSi5DJzY5bpZyeg9CPBkV9QDzJ1yFRlmi/j7WWRf3hYIWrOaJj5jsfwgc8GTHQ==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.46.4': resolution: {integrity: sha512-r0WKLSfFAK8ucG024v2yiLSJMedoWvk8yWqfNICX28NHDGeu3F/wBf8KG6mclghx4FsLePxJr/9N8rIj1PtCnw==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.46.4': resolution: {integrity: sha512-IaizpPP2UQU3MNyPH1u0Xxbm73D+4OupL0bjo4Hm0496e2wg3zuvoAIhubkD1NGy9fXILEExPQy87mweujEatA==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.46.4': resolution: {integrity: sha512-aCM29orANR0a8wk896p6UEgIfupReupnmISz6SUwMIwTGaTI8MuKdE0OD2LvEg8ondDyZdMvnaN3bW4nFbATPA==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.46.4': resolution: {integrity: sha512-0Xj1vZE3cbr/wda8d/m+UeuSL+TDpuozzdD4QaSzu/xSOMK0Su5RhIkF7KVHFQsobemUNHPLEcYllL7ZTCP/Cg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.46.4': resolution: {integrity: sha512-kM/orjpolfA5yxsx84kI6bnK47AAZuWxglGKcNmokw2yy9i5eHY5UAjcX45jemTJnfHAWo3/hOoRqEeeTdL5hw==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.46.4': resolution: {integrity: sha512-cNLH4psMEsWKILW0isbpQA2OvjXLbKvnkcJFmqAptPQbtLrobiapBJVj6RoIvg6UXVp5w0wnIfd/Q56cNpF+Ew==} @@ -3575,41 +3613,49 @@ packages: resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==} cpu: [arm64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==} cpu: [riscv64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==} cpu: [riscv64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==} cpu: [s390x] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==} cpu: [x64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==} cpu: [x64] os: [linux] + libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==} @@ -4181,6 +4227,12 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + csv-parse@5.6.0: + resolution: {integrity: sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==} + + csv-stringify@6.5.2: + resolution: {integrity: sha512-RFPahj0sXcmUyjrObAK+DOWtMvMIFV328n4qZJhgX3x2RqkQgOTU2mCUmiFR0CzM6AzChlRSUErjiJeEt8BaQA==} + damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} @@ -9703,6 +9755,19 @@ snapshots: - supports-color - typescript + '@payloadcms/plugin-import-export@3.77.0(@payloadcms/ui@3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)': + dependencies: + '@faceless-ui/modal': 3.0.0(react-dom@19.1.3(react@19.1.3))(react@19.1.3) + '@payloadcms/translations': 3.77.0 + '@payloadcms/ui': 3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2) + csv-parse: 5.6.0 + csv-stringify: 6.5.2 + payload: 3.77.0(graphql@16.11.0)(typescript@5.9.2) + qs-esm: 7.0.2 + transitivePeerDependencies: + - react + - react-dom + '@payloadcms/plugin-multi-tenant@3.77.0(@payloadcms/ui@3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))': dependencies: '@payloadcms/ui': 3.77.0(@types/react@19.1.11)(monaco-editor@0.52.2)(next@15.5.14(@babel/core@7.28.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(sass@1.77.4))(payload@3.77.0(graphql@16.11.0)(typescript@5.9.2))(react-dom@19.1.3(react@19.1.3))(react@19.1.3)(typescript@5.9.2) @@ -11792,6 +11857,10 @@ snapshots: csstype@3.1.3: {} + csv-parse@5.6.0: {} + + csv-stringify@6.5.2: {} + damerau-levenshtein@1.0.8: {} data-urls@5.0.0: diff --git a/src/app/(payload)/admin/importMap.js b/src/app/(payload)/admin/importMap.js index f9d0d61f..e430b46b 100644 --- a/src/app/(payload)/admin/importMap.js +++ b/src/app/(payload)/admin/importMap.js @@ -22,6 +22,8 @@ import { UnderlineFeatureClient as UnderlineFeatureClient_e70f5e05f09f93e00b997e import { BoldFeatureClient as BoldFeatureClient_e70f5e05f09f93e00b997edb1ef0c864 } from '@payloadcms/richtext-lexical/client' import { ItalicFeatureClient as ItalicFeatureClient_e70f5e05f09f93e00b997edb1ef0c864 } from '@payloadcms/richtext-lexical/client' import { CustomRowLabel as CustomRowLabel_0e293e8a29b93ac2dfde38afc0359696 } from '@/components/payload/RowLabel' +import { ExportListMenuItem as ExportListMenuItem_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { ImportListMenuItem as ImportListMenuItem_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { OverviewComponent as OverviewComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' import { MetaTitleComponent as MetaTitleComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' import { MetaDescriptionComponent as MetaDescriptionComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' @@ -33,10 +35,21 @@ import { SlugComponent as SlugComponent_92cc057d0a2abb4f6cf0307edf59f986 } from import { LinkGroupRowLabel as LinkGroupRowLabel_0e293e8a29b93ac2dfde38afc0359696 } from '@/components/payload/RowLabel' import { WatchTenantCollection as WatchTenantCollection_1d0591e3cf4f332c83a86da13a0de59a } from '@payloadcms/plugin-multi-tenant/client' import { ColorPickerFieldComponent as ColorPickerFieldComponent_95f860801d0381344387ae1d8e2f4d24 } from '@innovixx/payload-color-picker-field/components' +import { Page as Page_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { SortBy as SortBy_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { SortOrder as SortOrder_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { SelectionToUseField as SelectionToUseField_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { FieldsToExport as FieldsToExport_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { CollectionField as CollectionField_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { ExportPreview as ExportPreview_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { ExportSaveButton as ExportSaveButton_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { ImportPreview as ImportPreview_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { ImportSaveButton as ImportSaveButton_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { MaskedApiKeyField as MaskedApiKeyField_6e265c33523b378578f5880a6fcfdc0f } from '@/globals/Settings/tabs/MaskedApiKeyField' import { AIProviderRowLabel as AIProviderRowLabel_5a1b7be26cbec3efc44149c3bf55f6e7 } from '@/globals/Settings/tabs/RowLabel' import { GlobalViewRedirect as GlobalViewRedirect_d6d5f193a167989e2ee7d14202901e62 } from '@payloadcms/plugin-multi-tenant/rsc' import { TenantSelector as TenantSelector_d6d5f193a167989e2ee7d14202901e62 } from '@payloadcms/plugin-multi-tenant/rsc' +import { ImportExportProvider as ImportExportProvider_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { TenantSelectionProvider as TenantSelectionProvider_d6d5f193a167989e2ee7d14202901e62 } from '@payloadcms/plugin-multi-tenant/rsc' import { AdminErrorBoundary as AdminErrorBoundary_e5a9e14bdbe97e70ba60697217fe7688 } from '@payloadcms/plugin-sentry/client' import { S3ClientUploadHandler as S3ClientUploadHandler_f97aa6c64367fa259c5bc0567239ef24 } from '@payloadcms/storage-s3/client' @@ -67,6 +80,8 @@ export const importMap = { "@payloadcms/richtext-lexical/client#BoldFeatureClient": BoldFeatureClient_e70f5e05f09f93e00b997edb1ef0c864, "@payloadcms/richtext-lexical/client#ItalicFeatureClient": ItalicFeatureClient_e70f5e05f09f93e00b997edb1ef0c864, "@/components/payload/RowLabel#CustomRowLabel": CustomRowLabel_0e293e8a29b93ac2dfde38afc0359696, + "@payloadcms/plugin-import-export/rsc#ExportListMenuItem": ExportListMenuItem_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#ImportListMenuItem": ImportListMenuItem_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-seo/client#OverviewComponent": OverviewComponent_a8a977ebc872c5d5ea7ee689724c0860, "@payloadcms/plugin-seo/client#MetaTitleComponent": MetaTitleComponent_a8a977ebc872c5d5ea7ee689724c0860, "@payloadcms/plugin-seo/client#MetaDescriptionComponent": MetaDescriptionComponent_a8a977ebc872c5d5ea7ee689724c0860, @@ -78,10 +93,21 @@ export const importMap = { "@/components/payload/RowLabel#LinkGroupRowLabel": LinkGroupRowLabel_0e293e8a29b93ac2dfde38afc0359696, "@payloadcms/plugin-multi-tenant/client#WatchTenantCollection": WatchTenantCollection_1d0591e3cf4f332c83a86da13a0de59a, "@innovixx/payload-color-picker-field/components#ColorPickerFieldComponent": ColorPickerFieldComponent_95f860801d0381344387ae1d8e2f4d24, + "@payloadcms/plugin-import-export/rsc#Page": Page_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#SortBy": SortBy_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#SortOrder": SortOrder_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#SelectionToUseField": SelectionToUseField_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#FieldsToExport": FieldsToExport_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#CollectionField": CollectionField_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#ExportPreview": ExportPreview_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#ExportSaveButton": ExportSaveButton_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#ImportPreview": ImportPreview_cdf7e044479f899a31f804427d568b36, + "@payloadcms/plugin-import-export/rsc#ImportSaveButton": ImportSaveButton_cdf7e044479f899a31f804427d568b36, "@/globals/Settings/tabs/MaskedApiKeyField#MaskedApiKeyField": MaskedApiKeyField_6e265c33523b378578f5880a6fcfdc0f, "@/globals/Settings/tabs/RowLabel#AIProviderRowLabel": AIProviderRowLabel_5a1b7be26cbec3efc44149c3bf55f6e7, "@payloadcms/plugin-multi-tenant/rsc#GlobalViewRedirect": GlobalViewRedirect_d6d5f193a167989e2ee7d14202901e62, "@payloadcms/plugin-multi-tenant/rsc#TenantSelector": TenantSelector_d6d5f193a167989e2ee7d14202901e62, + "@payloadcms/plugin-import-export/rsc#ImportExportProvider": ImportExportProvider_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-multi-tenant/rsc#TenantSelectionProvider": TenantSelectionProvider_d6d5f193a167989e2ee7d14202901e62, "@payloadcms/plugin-sentry/client#AdminErrorBoundary": AdminErrorBoundary_e5a9e14bdbe97e70ba60697217fe7688, "@payloadcms/storage-s3/client#S3ClientUploadHandler": S3ClientUploadHandler_f97aa6c64367fa259c5bc0567239ef24, diff --git a/src/payload-types.ts b/src/payload-types.ts index 11a57aa4..dde2369b 100644 --- a/src/payload-types.ts +++ b/src/payload-types.ts @@ -79,6 +79,8 @@ export interface Config { partners: Partner; 'political-entities': PoliticalEntity; 'promise-status': PromiseStatus; + exports: Export; + imports: Import; 'payload-kv': PayloadKv; 'payload-jobs': PayloadJob; 'payload-locked-documents': PayloadLockedDocument; @@ -99,6 +101,8 @@ export interface Config { partners: PartnersSelect | PartnersSelect; 'political-entities': PoliticalEntitiesSelect | PoliticalEntitiesSelect; 'promise-status': PromiseStatusSelect | PromiseStatusSelect; + exports: ExportsSelect | ExportsSelect; + imports: ImportsSelect | ImportsSelect; 'payload-kv': PayloadKvSelect | PayloadKvSelect; 'payload-jobs': PayloadJobsSelect | PayloadJobsSelect; 'payload-locked-documents': PayloadLockedDocumentsSelect | PayloadLockedDocumentsSelect; @@ -138,6 +142,8 @@ export interface Config { updatePromiseStatus: TaskUpdatePromiseStatus; syncMeedanPromises: TaskSyncMeedanPromises; cleanupFailedJobs: TaskCleanupFailedJobs; + createCollectionExport: TaskCreateCollectionExport; + createCollectionImport: TaskCreateCollectionImport; inline: { input: unknown; output: unknown; @@ -923,6 +929,81 @@ export interface SiteSetting { updatedAt: string; createdAt: string; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "exports". + */ +export interface Export { + id: string; + name?: string | null; + format: 'csv' | 'json'; + limit?: number | null; + page?: number | null; + sort?: string | null; + sortOrder?: ('asc' | 'desc') | null; + locale?: ('all' | 'en' | 'fr') | null; + drafts?: ('yes' | 'no') | null; + selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null; + fields?: string[] | null; + collectionSlug: string; + where?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + updatedAt: string; + createdAt: string; + url?: string | null; + thumbnailURL?: string | null; + filename?: string | null; + mimeType?: string | null; + filesize?: number | null; + width?: number | null; + height?: number | null; + focalX?: number | null; + focalY?: number | null; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "imports". + */ +export interface Import { + id: string; + collectionSlug: string; + importMode?: ('create' | 'update' | 'upsert') | null; + matchField?: string | null; + status?: ('pending' | 'completed' | 'partial' | 'failed') | null; + summary?: { + imported?: number | null; + updated?: number | null; + total?: number | null; + issues?: number | null; + issueDetails?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + }; + updatedAt: string; + createdAt: string; + url?: string | null; + thumbnailURL?: string | null; + filename?: string | null; + mimeType?: string | null; + filesize?: number | null; + width?: number | null; + height?: number | null; + focalX?: number | null; + focalY?: number | null; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "payload-kv". @@ -1004,7 +1085,9 @@ export interface PayloadJob { | 'fetchPromiseStatuses' | 'updatePromiseStatus' | 'syncMeedanPromises' - | 'cleanupFailedJobs'; + | 'cleanupFailedJobs' + | 'createCollectionExport' + | 'createCollectionImport'; taskID: string; input?: | { @@ -1049,6 +1132,8 @@ export interface PayloadJob { | 'updatePromiseStatus' | 'syncMeedanPromises' | 'cleanupFailedJobs' + | 'createCollectionExport' + | 'createCollectionImport' ) | null; taskID?: string | null; @@ -1071,6 +1156,8 @@ export interface PayloadJob { | 'updatePromiseStatus' | 'syncMeedanPromises' | 'cleanupFailedJobs' + | 'createCollectionExport' + | 'createCollectionImport' ) | null; queue?: string | null; @@ -1709,6 +1796,65 @@ export interface PromiseStatusSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "exports_select". + */ +export interface ExportsSelect { + name?: T; + format?: T; + limit?: T; + page?: T; + sort?: T; + sortOrder?: T; + locale?: T; + drafts?: T; + selectionToUse?: T; + fields?: T; + collectionSlug?: T; + where?: T; + updatedAt?: T; + createdAt?: T; + url?: T; + thumbnailURL?: T; + filename?: T; + mimeType?: T; + filesize?: T; + width?: T; + height?: T; + focalX?: T; + focalY?: T; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "imports_select". + */ +export interface ImportsSelect { + collectionSlug?: T; + importMode?: T; + matchField?: T; + status?: T; + summary?: + | T + | { + imported?: T; + updated?: T; + total?: T; + issues?: T; + issueDetails?: T; + }; + updatedAt?: T; + createdAt?: T; + url?: T; + thumbnailURL?: T; + filename?: T; + mimeType?: T; + filesize?: T; + width?: T; + height?: T; + focalX?: T; + focalY?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "payload-kv_select". @@ -2830,6 +2976,58 @@ export interface TaskCleanupFailedJobs { input?: unknown; output?: unknown; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "TaskCreateCollectionExport". + */ +export interface TaskCreateCollectionExport { + input: { + name?: string | null; + format: 'csv' | 'json'; + limit?: number | null; + page?: number | null; + sort?: string | null; + sortOrder?: ('asc' | 'desc') | null; + locale?: ('all' | 'en' | 'fr') | null; + drafts?: ('yes' | 'no') | null; + selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null; + fields?: string[] | null; + collectionSlug: string; + where?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + id?: string | null; + batchSize?: number | null; + userID?: string | null; + userCollection?: string | null; + exportCollection?: string | null; + maxLimit?: number | null; + }; + output?: unknown; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "TaskCreateCollectionImport". + */ +export interface TaskCreateCollectionImport { + input: { + importId: string; + importCollection: string; + userID?: string | null; + userCollection?: string | null; + batchSize?: number | null; + debug?: boolean | null; + defaultVersionStatus?: ('draft' | 'published') | null; + maxLimit?: number | null; + }; + output?: unknown; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "WorkflowAirtableWorkflow". diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 3f7fc726..2125bd7f 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -7,6 +7,7 @@ import { capitalizeFirstLetter, isProd } from "@/utils/utils"; import { s3Storage } from "@payloadcms/storage-s3"; import { seoPlugin } from "@payloadcms/plugin-seo"; import { convertLexicalToPlaintext } from "@payloadcms/richtext-lexical/plaintext"; +import { importExportPlugin } from "@payloadcms/plugin-import-export"; const accessKeyId = process.env.S3_ACCESS_KEY_ID ?? ""; const bucket = process.env.S3_BUCKET ?? ""; @@ -15,6 +16,17 @@ const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY ?? ""; const s3Enabled = !!accessKeyId && !!region && !!secretAccessKey; export const plugins: Plugin[] = [ + importExportPlugin({ + debug: !isProd, + collections: [ + { + slug: "promises", + }, + { + slug: "ai-extractions", + }, + ], + }), multiTenantPlugin({ collections: { pages: {}, From 59a61c6d0e909ef2f937a14edc3f067fbfaeac8d Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:11:54 +0300 Subject: [PATCH 02/12] feat: add ai extraction export rows collection and sync logic Introduce a new collection `ai-extraction-export-rows` to provide flattened export data for AI extractions, enabling CSV exports via the import-export plugin. Add hooks to related collections (AIExtractions, Documents, PoliticalEntities, Tenants) to automatically synchronize export rows on changes. Include a migration to backfill existing data, a scheduled task for rebuilding, and comprehensive unit tests. Update package dependencies and fix import ordering. --- ...0000_backfill_ai_extraction_export_rows.ts | 17 + package.json | 2 +- pnpm-lock.yaml | 4 +- src/app/(payload)/admin/importMap.js | 4 +- src/collections/AIExtractionExportRows.ts | 135 ++++++ src/collections/AIExtractions.ts | 24 + src/collections/Documents.ts | 24 + src/collections/PoliticalEntities/index.ts | 24 + src/collections/Tenant/index.ts | 22 + src/collections/index.ts | 2 + src/lib/aiExtractionExportRows.ts | 452 ++++++++++++++++++ src/payload-types.ts | 101 ++++ src/plugins/index.ts | 23 +- src/tasks/index.ts | 2 + src/tasks/syncAIExtractionExportRows.ts | 30 ++ tests/int/aiExtractionExportRows.int.spec.ts | 116 +++++ 16 files changed, 975 insertions(+), 7 deletions(-) create mode 100644 migrations/20260423_000000_backfill_ai_extraction_export_rows.ts create mode 100644 src/collections/AIExtractionExportRows.ts create mode 100644 src/lib/aiExtractionExportRows.ts create mode 100644 src/tasks/syncAIExtractionExportRows.ts create mode 100644 tests/int/aiExtractionExportRows.int.spec.ts diff --git a/migrations/20260423_000000_backfill_ai_extraction_export_rows.ts b/migrations/20260423_000000_backfill_ai_extraction_export_rows.ts new file mode 100644 index 00000000..406238da --- /dev/null +++ b/migrations/20260423_000000_backfill_ai_extraction_export_rows.ts @@ -0,0 +1,17 @@ +import type { MigrateDownArgs, MigrateUpArgs } from "@payloadcms/db-mongodb"; +import { + AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + rebuildAllAIExtractionExportRows, +} from "../src/lib/aiExtractionExportRows"; + +export async function up({ payload }: MigrateUpArgs): Promise { + await rebuildAllAIExtractionExportRows({ payload }); +} + +export async function down({ payload }: MigrateDownArgs): Promise { + await payload.delete({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + overrideAccess: true, + where: {}, + }); +} diff --git a/package.json b/package.json index 4d36bcdc..b777bc4c 100644 --- a/package.json +++ b/package.json @@ -48,10 +48,10 @@ "@next/third-parties": "^15.5.3", "@payloadcms/db-mongodb": "3.77.0", "@payloadcms/email-nodemailer": "3.77.0", - "@payloadcms/plugin-import-export": "3.77.0", "@payloadcms/next": "3.77.0", "@payloadcms/payload-cloud": "3.77.0", "@payloadcms/plugin-cloud-storage": "3.77.0", + "@payloadcms/plugin-import-export": "3.77.0", "@payloadcms/plugin-multi-tenant": "3.77.0", "@payloadcms/plugin-sentry": "3.77.0", "@payloadcms/plugin-seo": "3.77.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47455898..b8d69fb5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11954,7 +11954,7 @@ snapshots: dom-helpers@5.2.1: dependencies: - '@babel/runtime': 7.28.3 + '@babel/runtime': 7.29.2 csstype: 3.1.3 dom-serializer@2.0.0: @@ -13814,7 +13814,7 @@ snapshots: react-transition-group@4.4.5(react-dom@19.1.3(react@19.1.3))(react@19.1.3): dependencies: - '@babel/runtime': 7.28.3 + '@babel/runtime': 7.29.2 dom-helpers: 5.2.1 loose-envify: 1.4.0 prop-types: 15.8.1 diff --git a/src/app/(payload)/admin/importMap.js b/src/app/(payload)/admin/importMap.js index e430b46b..e36b2166 100644 --- a/src/app/(payload)/admin/importMap.js +++ b/src/app/(payload)/admin/importMap.js @@ -23,12 +23,12 @@ import { BoldFeatureClient as BoldFeatureClient_e70f5e05f09f93e00b997edb1ef0c864 import { ItalicFeatureClient as ItalicFeatureClient_e70f5e05f09f93e00b997edb1ef0c864 } from '@payloadcms/richtext-lexical/client' import { CustomRowLabel as CustomRowLabel_0e293e8a29b93ac2dfde38afc0359696 } from '@/components/payload/RowLabel' import { ExportListMenuItem as ExportListMenuItem_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' -import { ImportListMenuItem as ImportListMenuItem_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { OverviewComponent as OverviewComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' import { MetaTitleComponent as MetaTitleComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' import { MetaDescriptionComponent as MetaDescriptionComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' import { MetaImageComponent as MetaImageComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' import { PreviewComponent as PreviewComponent_a8a977ebc872c5d5ea7ee689724c0860 } from '@payloadcms/plugin-seo/client' +import { ImportListMenuItem as ImportListMenuItem_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { TenantField as TenantField_1d0591e3cf4f332c83a86da13a0de59a } from '@payloadcms/plugin-multi-tenant/client' import { CollectionTenantFieldCell as CollectionTenantFieldCell_14d997eb17eff525bd3fc0edd0556c55 } from '@/components/payload/CollectionTenantFieldCell' import { SlugComponent as SlugComponent_92cc057d0a2abb4f6cf0307edf59f986 } from '@/fields/slug/SlugComponent' @@ -81,12 +81,12 @@ export const importMap = { "@payloadcms/richtext-lexical/client#ItalicFeatureClient": ItalicFeatureClient_e70f5e05f09f93e00b997edb1ef0c864, "@/components/payload/RowLabel#CustomRowLabel": CustomRowLabel_0e293e8a29b93ac2dfde38afc0359696, "@payloadcms/plugin-import-export/rsc#ExportListMenuItem": ExportListMenuItem_cdf7e044479f899a31f804427d568b36, - "@payloadcms/plugin-import-export/rsc#ImportListMenuItem": ImportListMenuItem_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-seo/client#OverviewComponent": OverviewComponent_a8a977ebc872c5d5ea7ee689724c0860, "@payloadcms/plugin-seo/client#MetaTitleComponent": MetaTitleComponent_a8a977ebc872c5d5ea7ee689724c0860, "@payloadcms/plugin-seo/client#MetaDescriptionComponent": MetaDescriptionComponent_a8a977ebc872c5d5ea7ee689724c0860, "@payloadcms/plugin-seo/client#MetaImageComponent": MetaImageComponent_a8a977ebc872c5d5ea7ee689724c0860, "@payloadcms/plugin-seo/client#PreviewComponent": PreviewComponent_a8a977ebc872c5d5ea7ee689724c0860, + "@payloadcms/plugin-import-export/rsc#ImportListMenuItem": ImportListMenuItem_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-multi-tenant/client#TenantField": TenantField_1d0591e3cf4f332c83a86da13a0de59a, "@/components/payload/CollectionTenantFieldCell#CollectionTenantFieldCell": CollectionTenantFieldCell_14d997eb17eff525bd3fc0edd0556c55, "@/fields/slug/SlugComponent#SlugComponent": SlugComponent_92cc057d0a2abb4f6cf0307edf59f986, diff --git a/src/collections/AIExtractionExportRows.ts b/src/collections/AIExtractionExportRows.ts new file mode 100644 index 00000000..f33a5c0e --- /dev/null +++ b/src/collections/AIExtractionExportRows.ts @@ -0,0 +1,135 @@ +import { AI_EXTRACTION_EXPORT_ROWS_COLLECTION } from "@/lib/aiExtractionExportRows"; +import { CollectionConfig, Field } from "payload"; + +const disabledExportRelationship = { + "plugin-import-export": { + disabled: true, + }, +}; + +const relationshipField = ({ + name, + relationTo, +}: { + name: string; + relationTo: + | "ai-extractions" + | "documents" + | "political-entities" + | "promise-status" + | "tenants"; +}): Field => ({ + name, + type: "relationship", + relationTo, + admin: { + hidden: true, + }, + custom: disabledExportRelationship, +}); + +const textField = (name: string, label: string): Field => ({ + name, + type: "text", + label, + admin: { + readOnly: true, + }, +}); + +export const AIExtractionExportRows: CollectionConfig = { + slug: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + labels: { + singular: "AI Extraction Export Row", + plural: "AI Extraction Export Rows", + }, + admin: { + defaultColumns: [ + "tenantName", + "politicalEntityName", + "documentTitle", + "category", + "summary", + "statusLabel", + ], + group: { + en: "Documents", + fr: "Documents", + }, + useAsTitle: "summary", + }, + access: { + create: () => false, + delete: () => false, + read: ({ req }) => Boolean(req.user), + update: () => false, + }, + fields: [ + { + name: "uniqueKey", + type: "text", + unique: true, + index: true, + required: true, + admin: { + hidden: true, + readOnly: true, + }, + custom: disabledExportRelationship, + }, + relationshipField({ + name: "aiExtraction", + relationTo: "ai-extractions", + }), + relationshipField({ + name: "document", + relationTo: "documents", + }), + relationshipField({ + name: "politicalEntity", + relationTo: "political-entities", + }), + relationshipField({ + name: "tenant", + relationTo: "tenants", + }), + relationshipField({ + name: "status", + relationTo: "promise-status", + }), + textField("tenantId", "Tenant ID"), + textField("tenantName", "Tenant Name"), + textField("tenantCountry", "Tenant Country"), + textField("tenantLocale", "Tenant Locale"), + textField("politicalEntityId", "Political Entity ID"), + textField("politicalEntityName", "Political Entity Name"), + textField("politicalEntitySlug", "Political Entity Slug"), + textField("politicalEntityPosition", "Political Entity Position"), + textField("documentId", "Document ID"), + textField("documentTitle", "Document Title"), + textField("documentUrl", "Document URL"), + textField("documentLanguage", "Document Language"), + textField("documentType", "Document Type"), + textField("documentAirtableID", "Document Airtable ID"), + textField("aiExtractionId", "AI Extraction ID"), + textField("aiExtractionTitle", "AI Extraction Title"), + textField("extractionRowId", "Extraction Row ID"), + textField("uniqueId", "Unique ID"), + textField("category", "Category"), + textField("summary", "Summary"), + { + name: "source", + type: "textarea", + label: "Source", + admin: { + readOnly: true, + }, + }, + textField("statusId", "Status ID"), + textField("statusLabel", "Status Label"), + textField("statusMeedanId", "Status Meedan ID"), + textField("checkMediaId", "CheckMedia ID"), + textField("checkMediaURL", "CheckMedia URL"), + textField("uploadError", "Upload Error"), + ], +}; diff --git a/src/collections/AIExtractions.ts b/src/collections/AIExtractions.ts index 8eb0d1ce..2979da9a 100644 --- a/src/collections/AIExtractions.ts +++ b/src/collections/AIExtractions.ts @@ -1,3 +1,7 @@ +import { + deleteAIExtractionExportRowsForAIExtraction, + syncAIExtractionExportRows, +} from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; export const AIExtractions: CollectionConfig = { @@ -22,6 +26,26 @@ export const AIExtractions: CollectionConfig = { access: { read: () => true, }, + hooks: { + afterChange: [ + async ({ doc, req }) => { + await syncAIExtractionExportRows({ + aiExtractionId: String(doc.id), + payload: req.payload, + }); + return doc; + }, + ], + afterDelete: [ + async ({ doc, req }) => { + await deleteAIExtractionExportRowsForAIExtraction({ + aiExtractionId: String(doc.id), + payload: req.payload, + }); + return doc; + }, + ], + }, fields: [ { name: "title", diff --git a/src/collections/Documents.ts b/src/collections/Documents.ts index e8d141c4..76d0945d 100644 --- a/src/collections/Documents.ts +++ b/src/collections/Documents.ts @@ -1,4 +1,8 @@ import { airtableID } from "@/fields/airtableID"; +import { + deleteAIExtractionExportRowsForDocument, + syncAIExtractionExportRowsForDocument, +} from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; export const Documents: CollectionConfig = { @@ -16,6 +20,26 @@ export const Documents: CollectionConfig = { access: { read: () => true, }, + hooks: { + afterChange: [ + async ({ doc, req }) => { + await syncAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload: req.payload, + }); + return doc; + }, + ], + afterDelete: [ + async ({ doc, req }) => { + await deleteAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload: req.payload, + }); + return doc; + }, + ], + }, admin: { defaultColumns: ["title", "politicalEntity", "language", "type"], useAsTitle: "title", diff --git a/src/collections/PoliticalEntities/index.ts b/src/collections/PoliticalEntities/index.ts index 52bfa59e..7fc12095 100644 --- a/src/collections/PoliticalEntities/index.ts +++ b/src/collections/PoliticalEntities/index.ts @@ -1,5 +1,9 @@ import { airtableID } from "@/fields/airtableID"; import { image } from "@/fields/image"; +import { + deleteAIExtractionExportRowsForPoliticalEntity, + syncAIExtractionExportRowsForPoliticalEntity, +} from "@/lib/aiExtractionExportRows"; import { slugField } from "@/fields/slug"; import { CollectionConfig } from "payload"; import { ensureUniqueSlug } from "./hooks/ensureUniqueSlug"; @@ -33,6 +37,26 @@ export const PoliticalEntities: CollectionConfig = { "periodTo", ], }, + hooks: { + afterChange: [ + async ({ doc, req }) => { + await syncAIExtractionExportRowsForPoliticalEntity({ + payload: req.payload, + politicalEntityId: String(doc.id), + }); + return doc; + }, + ], + afterDelete: [ + async ({ doc, req }) => { + await deleteAIExtractionExportRowsForPoliticalEntity({ + payload: req.payload, + politicalEntityId: String(doc.id), + }); + return doc; + }, + ], + }, fields: [ { name: "name", diff --git a/src/collections/Tenant/index.ts b/src/collections/Tenant/index.ts index f5418370..5a1d02d2 100644 --- a/src/collections/Tenant/index.ts +++ b/src/collections/Tenant/index.ts @@ -1,5 +1,9 @@ import { countriesByContinent, getCountryFlag } from "@/data/countries"; import { airtableID } from "@/fields/airtableID"; +import { + deleteAIExtractionExportRowsForTenant, + syncAIExtractionExportRowsForTenant, +} from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; const africanCountries = countriesByContinent("Africa"); @@ -89,6 +93,24 @@ export const Tenants: CollectionConfig = { airtableID(), ], hooks: { + afterChange: [ + async ({ doc, req }) => { + await syncAIExtractionExportRowsForTenant({ + payload: req.payload, + tenantId: String(doc.id), + }); + return doc; + }, + ], + afterDelete: [ + async ({ doc, req }) => { + await deleteAIExtractionExportRowsForTenant({ + payload: req.payload, + tenantId: String(doc.id), + }); + return doc; + }, + ], afterRead: [ async ({ doc }) => { doc.flag = getCountryFlag(doc.country); diff --git a/src/collections/index.ts b/src/collections/index.ts index dee178d3..7169fbfa 100644 --- a/src/collections/index.ts +++ b/src/collections/index.ts @@ -3,6 +3,7 @@ import { Users } from "./Users"; import { Media } from "./Media"; import { Documents } from "./Documents"; import { AIExtractions } from "./AIExtractions"; +import { AIExtractionExportRows } from "./AIExtractionExportRows"; import { Promises } from "./Promises"; import { Tenants } from "./Tenant"; import { Pages } from "./Pages"; @@ -15,6 +16,7 @@ import { GlobalPages } from "./Pages/GlobalPages"; export const collections: CollectionConfig[] = [ Documents, AIExtractions, + AIExtractionExportRows, Promises, Media, Pages, diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts new file mode 100644 index 00000000..c24d3eb5 --- /dev/null +++ b/src/lib/aiExtractionExportRows.ts @@ -0,0 +1,452 @@ +import type { + AiExtraction, + Document as PayloadDocument, + PoliticalEntity, + PromiseStatus, + Tenant, +} from "@/payload-types"; +import type { Payload, Where } from "payload"; + +export const AI_EXTRACTION_EXPORT_ROWS_COLLECTION = + "ai-extraction-export-rows" as const; + +type ExtractionItem = NonNullable< + NonNullable[number] +>; + +export type AIExtractionExportRowData = { + uniqueKey: string; + aiExtraction?: string; + aiExtractionId: string; + aiExtractionTitle: string; + extractionRowId: string; + uniqueId: string; + category: string; + summary: string; + source: string; + uploadError: string; + checkMediaId: string; + checkMediaURL: string; + status?: string; + statusId: string; + statusLabel: string; + statusMeedanId: string; + document?: string; + documentId: string; + documentTitle: string; + documentUrl: string; + documentLanguage: string; + documentType: string; + documentAirtableID: string; + politicalEntity?: string; + politicalEntityId: string; + politicalEntityName: string; + politicalEntitySlug: string; + politicalEntityPosition: string; + tenant?: string; + tenantId: string; + tenantName: string; + tenantCountry: string; + tenantLocale: string; +}; + +type ExistingExportRow = { + id: string; + uniqueKey?: string | null; +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +const getId = (value: unknown): string => { + if (typeof value === "string") { + return value; + } + + if (isRecord(value) && typeof value.id === "string") { + return value.id; + } + + return ""; +}; + +const getString = (value: unknown, key: string): string => { + if (!isRecord(value)) { + return ""; + } + + const fieldValue = value[key]; + if (typeof fieldValue === "string") { + return fieldValue; + } + + if (typeof fieldValue === "number" || typeof fieldValue === "boolean") { + return String(fieldValue); + } + + return ""; +}; + +const getRelationshipRecord = ( + value: string | T | null | undefined, +): T | null => (isRecord(value) ? (value as T) : null); + +const getDocumentUrl = (document: PayloadDocument | null): string => { + if (!document) { + return ""; + } + + if (document.url) { + return document.url; + } + + const [firstDocURL] = document.docURLs ?? []; + return firstDocURL?.url ?? ""; +}; + +const getExtractionRowId = ( + extraction: ExtractionItem, + index: number, +): string => extraction.uniqueId ?? extraction.id ?? String(index + 1); + +export const buildAIExtractionExportRows = ( + extractionDoc: AiExtraction, +): AIExtractionExportRowData[] => { + const document = getRelationshipRecord( + extractionDoc.document, + ); + const politicalEntity = getRelationshipRecord( + document?.politicalEntity, + ); + const tenant = getRelationshipRecord(politicalEntity?.tenant); + const aiExtractionId = String(extractionDoc.id); + + return (extractionDoc.extractions ?? []).map((extraction, index) => { + const status = getRelationshipRecord(extraction.Status); + const extractionRowId = getExtractionRowId(extraction, index); + + return { + uniqueKey: `${aiExtractionId}:${extractionRowId}`, + aiExtraction: aiExtractionId, + aiExtractionId, + aiExtractionTitle: extractionDoc.title ?? "", + extractionRowId, + uniqueId: extraction.uniqueId ?? "", + category: extraction.category, + summary: extraction.summary, + source: extraction.source, + uploadError: extraction.uploadError ?? "", + checkMediaId: extraction.checkMediaId ?? "", + checkMediaURL: extraction.checkMediaURL ?? "", + status: getId(extraction.Status) || undefined, + statusId: getId(extraction.Status), + statusLabel: status?.label ?? "", + statusMeedanId: status?.meedanId ?? "", + document: getId(extractionDoc.document) || undefined, + documentId: getId(extractionDoc.document), + documentTitle: document?.title ?? "", + documentUrl: getDocumentUrl(document), + documentLanguage: document?.language ?? "", + documentType: document?.type ?? "", + documentAirtableID: document?.airtableID ?? "", + politicalEntity: getId(document?.politicalEntity) || undefined, + politicalEntityId: getId(document?.politicalEntity), + politicalEntityName: politicalEntity?.name ?? "", + politicalEntitySlug: politicalEntity?.slug ?? "", + politicalEntityPosition: politicalEntity?.position ?? "", + tenant: getId(politicalEntity?.tenant) || undefined, + tenantId: getId(politicalEntity?.tenant), + tenantName: tenant?.name ?? "", + tenantCountry: getString(tenant, "country"), + tenantLocale: getString(tenant, "locale"), + }; + }); +}; + +const findExistingRows = async ({ + aiExtractionId, + payload, +}: { + aiExtractionId: string; + payload: Payload; +}): Promise => { + const { docs } = await payload.find({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + depth: 0, + limit: 0, + overrideAccess: true, + where: { + aiExtractionId: { + equals: aiExtractionId, + }, + }, + }); + + return docs as ExistingExportRow[]; +}; + +const deleteRowsWhere = async ({ + payload, + where, +}: { + payload: Payload; + where: Where; +}) => { + const { docs } = await payload.find({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + depth: 0, + limit: 0, + overrideAccess: true, + where, + }); + + for (const row of docs as ExistingExportRow[]) { + await payload.delete({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + id: row.id, + overrideAccess: true, + }); + } +}; + +export const syncAIExtractionExportRows = async ({ + aiExtractionId, + payload, +}: { + aiExtractionId: string; + payload: Payload; +}) => { + const extractionDoc = (await payload.findByID({ + collection: "ai-extractions", + depth: 3, + id: aiExtractionId, + overrideAccess: true, + })) as AiExtraction; + + const rows = buildAIExtractionExportRows(extractionDoc); + const existingRows = await findExistingRows({ aiExtractionId, payload }); + const existingRowsByKey = new Map( + existingRows + .filter((row) => row.uniqueKey) + .map((row) => [row.uniqueKey as string, row]), + ); + const syncedKeys = new Set(); + + for (const row of rows) { + const existingRow = existingRowsByKey.get(row.uniqueKey); + syncedKeys.add(row.uniqueKey); + + if (existingRow) { + await payload.update({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + data: row, + id: existingRow.id, + overrideAccess: true, + }); + continue; + } + + await payload.create({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + data: row, + overrideAccess: true, + }); + } + + for (const row of existingRows) { + if (row.uniqueKey && syncedKeys.has(row.uniqueKey)) { + continue; + } + + await payload.delete({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + id: row.id, + overrideAccess: true, + }); + } +}; + +export const deleteAIExtractionExportRowsForAIExtraction = async ({ + aiExtractionId, + payload, +}: { + aiExtractionId: string; + payload: Payload; +}) => + deleteRowsWhere({ + payload, + where: { + aiExtractionId: { + equals: aiExtractionId, + }, + }, + }); + +export const deleteAIExtractionExportRowsForDocument = async ({ + documentId, + payload, +}: { + documentId: string; + payload: Payload; +}) => + deleteRowsWhere({ + payload, + where: { + documentId: { + equals: documentId, + }, + }, + }); + +export const deleteAIExtractionExportRowsForPoliticalEntity = async ({ + payload, + politicalEntityId, +}: { + payload: Payload; + politicalEntityId: string; +}) => + deleteRowsWhere({ + payload, + where: { + politicalEntityId: { + equals: politicalEntityId, + }, + }, + }); + +export const deleteAIExtractionExportRowsForTenant = async ({ + payload, + tenantId, +}: { + payload: Payload; + tenantId: string; +}) => + deleteRowsWhere({ + payload, + where: { + tenantId: { + equals: tenantId, + }, + }, + }); + +export const syncAIExtractionExportRowsForDocument = async ({ + documentId, + payload, +}: { + documentId: string; + payload: Payload; +}) => { + const { docs } = await payload.find({ + collection: "ai-extractions", + depth: 0, + limit: 0, + overrideAccess: true, + where: { + document: { + equals: documentId, + }, + }, + }); + + for (const doc of docs as Pick[]) { + await syncAIExtractionExportRows({ + aiExtractionId: String(doc.id), + payload, + }); + } +}; + +export const syncAIExtractionExportRowsForPoliticalEntity = async ({ + payload, + politicalEntityId, +}: { + payload: Payload; + politicalEntityId: string; +}) => { + const { docs } = await payload.find({ + collection: "documents", + depth: 0, + limit: 0, + overrideAccess: true, + where: { + politicalEntity: { + equals: politicalEntityId, + }, + }, + }); + + for (const doc of docs as Pick[]) { + await syncAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload, + }); + } +}; + +export const syncAIExtractionExportRowsForTenant = async ({ + payload, + tenantId, +}: { + payload: Payload; + tenantId: string; +}) => { + const { docs } = await payload.find({ + collection: "political-entities", + depth: 0, + limit: 0, + overrideAccess: true, + where: { + tenant: { + equals: tenantId, + }, + }, + }); + + for (const entity of docs as Pick[]) { + await syncAIExtractionExportRowsForPoliticalEntity({ + payload, + politicalEntityId: String(entity.id), + }); + } +}; + +export const rebuildAllAIExtractionExportRows = async ({ + batchSize = 100, + payload, +}: { + batchSize?: number; + payload: Payload; +}) => { + await deleteRowsWhere({ + payload, + where: {}, + }); + + let hasNextPage = true; + let page = 1; + let processed = 0; + + while (hasNextPage) { + const { docs, hasNextPage: nextPage } = await payload.find({ + collection: "ai-extractions", + depth: 0, + limit: batchSize, + overrideAccess: true, + page, + }); + + for (const doc of docs as Pick[]) { + await syncAIExtractionExportRows({ + aiExtractionId: String(doc.id), + payload, + }); + processed += 1; + } + + hasNextPage = nextPage; + page += 1; + } + + return { processed }; +}; diff --git a/src/payload-types.ts b/src/payload-types.ts index dde2369b..f757f12e 100644 --- a/src/payload-types.ts +++ b/src/payload-types.ts @@ -69,6 +69,7 @@ export interface Config { collections: { documents: Document; 'ai-extractions': AiExtraction; + 'ai-extraction-export-rows': AiExtractionExportRow; promises: Promise; media: Media; pages: Page; @@ -91,6 +92,7 @@ export interface Config { collectionsSelect: { documents: DocumentsSelect | DocumentsSelect; 'ai-extractions': AiExtractionsSelect | AiExtractionsSelect; + 'ai-extraction-export-rows': AiExtractionExportRowsSelect | AiExtractionExportRowsSelect; promises: PromisesSelect | PromisesSelect; media: MediaSelect | MediaSelect; pages: PagesSelect | PagesSelect; @@ -141,6 +143,7 @@ export interface Config { fetchPromiseStatuses: TaskFetchPromiseStatuses; updatePromiseStatus: TaskUpdatePromiseStatus; syncMeedanPromises: TaskSyncMeedanPromises; + syncAIExtractionExportRows: TaskSyncAIExtractionExportRows; cleanupFailedJobs: TaskCleanupFailedJobs; createCollectionExport: TaskCreateCollectionExport; createCollectionImport: TaskCreateCollectionImport; @@ -396,6 +399,48 @@ export interface PromiseStatus { updatedAt: string; createdAt: string; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "ai-extraction-export-rows". + */ +export interface AiExtractionExportRow { + id: string; + uniqueKey: string; + aiExtraction?: (string | null) | AiExtraction; + document?: (string | null) | Document; + politicalEntity?: (string | null) | PoliticalEntity; + tenant?: (string | null) | Tenant; + status?: (string | null) | PromiseStatus; + tenantId?: string | null; + tenantName?: string | null; + tenantCountry?: string | null; + tenantLocale?: string | null; + politicalEntityId?: string | null; + politicalEntityName?: string | null; + politicalEntitySlug?: string | null; + politicalEntityPosition?: string | null; + documentId?: string | null; + documentTitle?: string | null; + documentUrl?: string | null; + documentLanguage?: string | null; + documentType?: string | null; + documentAirtableID?: string | null; + aiExtractionId?: string | null; + aiExtractionTitle?: string | null; + extractionRowId?: string | null; + uniqueId?: string | null; + category?: string | null; + summary?: string | null; + source?: string | null; + statusId?: string | null; + statusLabel?: string | null; + statusMeedanId?: string | null; + checkMediaId?: string | null; + checkMediaURL?: string | null; + uploadError?: string | null; + updatedAt: string; + createdAt: string; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "promises". @@ -1085,6 +1130,7 @@ export interface PayloadJob { | 'fetchPromiseStatuses' | 'updatePromiseStatus' | 'syncMeedanPromises' + | 'syncAIExtractionExportRows' | 'cleanupFailedJobs' | 'createCollectionExport' | 'createCollectionImport'; @@ -1131,6 +1177,7 @@ export interface PayloadJob { | 'fetchPromiseStatuses' | 'updatePromiseStatus' | 'syncMeedanPromises' + | 'syncAIExtractionExportRows' | 'cleanupFailedJobs' | 'createCollectionExport' | 'createCollectionImport' @@ -1155,6 +1202,7 @@ export interface PayloadJob { | 'fetchPromiseStatuses' | 'updatePromiseStatus' | 'syncMeedanPromises' + | 'syncAIExtractionExportRows' | 'cleanupFailedJobs' | 'createCollectionExport' | 'createCollectionImport' @@ -1190,6 +1238,10 @@ export interface PayloadLockedDocument { relationTo: 'ai-extractions'; value: string | AiExtraction; } | null) + | ({ + relationTo: 'ai-extraction-export-rows'; + value: string | AiExtractionExportRow; + } | null) | ({ relationTo: 'promises'; value: string | Promise; @@ -1323,6 +1375,47 @@ export interface AiExtractionsSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "ai-extraction-export-rows_select". + */ +export interface AiExtractionExportRowsSelect { + uniqueKey?: T; + aiExtraction?: T; + document?: T; + politicalEntity?: T; + tenant?: T; + status?: T; + tenantId?: T; + tenantName?: T; + tenantCountry?: T; + tenantLocale?: T; + politicalEntityId?: T; + politicalEntityName?: T; + politicalEntitySlug?: T; + politicalEntityPosition?: T; + documentId?: T; + documentTitle?: T; + documentUrl?: T; + documentLanguage?: T; + documentType?: T; + documentAirtableID?: T; + aiExtractionId?: T; + aiExtractionTitle?: T; + extractionRowId?: T; + uniqueId?: T; + category?: T; + summary?: T; + source?: T; + statusId?: T; + statusLabel?: T; + statusMeedanId?: T; + checkMediaId?: T; + checkMediaURL?: T; + uploadError?: T; + updatedAt?: T; + createdAt?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "promises_select". @@ -2968,6 +3061,14 @@ export interface TaskSyncMeedanPromises { input?: unknown; output?: unknown; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "TaskSyncAIExtractionExportRows". + */ +export interface TaskSyncAIExtractionExportRows { + input?: unknown; + output?: unknown; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "TaskCleanupFailedJobs". diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 2125bd7f..4ed11846 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -2,12 +2,12 @@ import { Plugin } from "payload"; import { sentryPlugin } from "@payloadcms/plugin-sentry"; import * as Sentry from "@sentry/nextjs"; import { multiTenantPlugin } from "@payloadcms/plugin-multi-tenant"; +import { importExportPlugin } from "@payloadcms/plugin-import-export"; import { Config } from "@/payload-types"; import { capitalizeFirstLetter, isProd } from "@/utils/utils"; import { s3Storage } from "@payloadcms/storage-s3"; import { seoPlugin } from "@payloadcms/plugin-seo"; import { convertLexicalToPlaintext } from "@payloadcms/richtext-lexical/plaintext"; -import { importExportPlugin } from "@payloadcms/plugin-import-export"; const accessKeyId = process.env.S3_ACCESS_KEY_ID ?? ""; const bucket = process.env.S3_BUCKET ?? ""; @@ -23,9 +23,28 @@ export const plugins: Plugin[] = [ slug: "promises", }, { - slug: "ai-extractions", + slug: "ai-extraction-export-rows", + export: { + format: "csv", + }, + import: false, }, ], + overrideExportCollection: ({ collection }) => ({ + ...collection, + access: { + ...collection.access, + create: ({ req }) => Boolean(req.user), + read: ({ req }) => Boolean(req.user), + }, + admin: { + ...collection.admin, + group: { + en: "Documents", + fr: "Documents", + }, + }, + }), }), multiTenantPlugin({ collections: { diff --git a/src/tasks/index.ts b/src/tasks/index.ts index 71013a98..c7a0aad6 100644 --- a/src/tasks/index.ts +++ b/src/tasks/index.ts @@ -10,6 +10,7 @@ import { FetchPromiseStatuses } from "./fetchMeedanPromiseStatus"; import { UpdatePromiseStatus } from "./updatePromiseStatus"; import { SyncMeedanPromises } from "./syncMeedanPromises"; import { CleanupFailedJobs } from "./cleanupFailedJobs"; +import { SyncAIExtractionExportRows } from "./syncAIExtractionExportRows"; export const tasks: TaskConfig[] = [ CreateTenantFromAirtable, @@ -22,5 +23,6 @@ export const tasks: TaskConfig[] = [ FetchPromiseStatuses, UpdatePromiseStatus, SyncMeedanPromises, + SyncAIExtractionExportRows, CleanupFailedJobs, ]; diff --git a/src/tasks/syncAIExtractionExportRows.ts b/src/tasks/syncAIExtractionExportRows.ts new file mode 100644 index 00000000..2a63d464 --- /dev/null +++ b/src/tasks/syncAIExtractionExportRows.ts @@ -0,0 +1,30 @@ +import { rebuildAllAIExtractionExportRows } from "@/lib/aiExtractionExportRows"; +import { TaskConfig } from "payload"; +import { getTaskLogger, withTaskTracing } from "./utils"; + +export const SyncAIExtractionExportRows: TaskConfig = { + slug: "syncAIExtractionExportRows", + label: "Sync AI Extraction Export Rows", + handler: withTaskTracing( + "syncAIExtractionExportRows", + async ({ req, input }) => { + const logger = getTaskLogger(req, "syncAIExtractionExportRows", input); + + logger.info( + "syncAIExtractionExportRows:: Rebuilding AI extraction export rows", + ); + + const result = await rebuildAllAIExtractionExportRows({ + payload: req.payload, + }); + + logger.info({ + message: + "syncAIExtractionExportRows:: Completed AI extraction export row rebuild", + processed: result.processed, + }); + + return { output: result }; + }, + ), +}; diff --git a/tests/int/aiExtractionExportRows.int.spec.ts b/tests/int/aiExtractionExportRows.int.spec.ts new file mode 100644 index 00000000..c6b039ea --- /dev/null +++ b/tests/int/aiExtractionExportRows.int.spec.ts @@ -0,0 +1,116 @@ +import { buildAIExtractionExportRows } from "@/lib/aiExtractionExportRows"; +import type { AiExtraction } from "@/payload-types"; +import { describe, expect, it } from "vitest"; + +describe("AI extraction export rows", () => { + it("creates one flat export row per extraction entry", () => { + const extractionDoc = { + id: "ai-extraction-1", + title: "Campaign Manifesto", + document: { + id: "document-1", + title: "Manifesto PDF", + url: "https://example.com/manifesto.pdf", + language: "en", + type: "promise", + airtableID: "rec-document", + politicalEntity: { + id: "political-entity-1", + name: "Jane Leader", + slug: "jane-leader", + position: "President", + tenant: { + id: "tenant-1", + name: "Kenya", + country: "KEN", + locale: "en", + }, + }, + }, + extractions: [ + { + id: "row-1", + category: "Health", + summary: "Build new clinics", + source: "Clinic source quote", + uniqueId: "unique-1", + checkMediaId: "check-1", + checkMediaURL: "https://check.example.com/media/check-1", + Status: { + id: "status-1", + label: "In Progress", + meedanId: "meedan-status-1", + }, + }, + { + id: "row-2", + category: "Education", + summary: "Hire teachers", + source: "Teacher source quote", + uploadError: "Upload failed", + }, + ], + } as AiExtraction; + + const rows = buildAIExtractionExportRows(extractionDoc); + + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + uniqueKey: "ai-extraction-1:unique-1", + tenantId: "tenant-1", + tenantName: "Kenya", + tenantCountry: "KEN", + politicalEntityId: "political-entity-1", + politicalEntityName: "Jane Leader", + documentId: "document-1", + documentTitle: "Manifesto PDF", + documentUrl: "https://example.com/manifesto.pdf", + aiExtractionId: "ai-extraction-1", + extractionRowId: "unique-1", + category: "Health", + summary: "Build new clinics", + statusId: "status-1", + statusLabel: "In Progress", + statusMeedanId: "meedan-status-1", + checkMediaId: "check-1", + checkMediaURL: "https://check.example.com/media/check-1", + }); + expect(rows[1]).toMatchObject({ + uniqueKey: "ai-extraction-1:row-2", + category: "Education", + statusId: "", + statusLabel: "", + uploadError: "Upload failed", + }); + }); + + it("falls back to docURLs and safely handles unresolved relationships", () => { + const extractionDoc = { + id: "ai-extraction-2", + document: { + id: "document-2", + title: "Airtable attachment", + docURLs: [{ url: "https://example.com/attachment.pdf" }], + politicalEntity: "political-entity-2", + }, + extractions: [ + { + category: "Jobs", + summary: "Create jobs", + source: "Jobs quote", + }, + ], + } as AiExtraction; + + const [row] = buildAIExtractionExportRows(extractionDoc); + + expect(row).toMatchObject({ + uniqueKey: "ai-extraction-2:1", + documentUrl: "https://example.com/attachment.pdf", + politicalEntityId: "political-entity-2", + politicalEntityName: "", + tenantId: "", + tenantName: "", + }); + }); +}); From 2dfa0ba191832f1482e55462143b00d1e5713542 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 23 Apr 2026 18:15:44 +0300 Subject: [PATCH 03/12] ci: remove Claude Code Review workflow This workflow was using a third-party action that is no longer needed. Removing it simplifies our CI setup and reduces dependency on external services. --- .github/workflows/claude-code-review.yml | 39 ------------------------ 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/claude-code-review.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml deleted file mode 100644 index 02157367..00000000 --- a/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Claude Code Review - -on: - pull_request: - types: [opened, synchronize, ready_for_review, reopened] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" - -jobs: - claude-review: - if: github.event.pull_request.head.repo.fork == false && github.event.pull_request.user.type != 'Bot' - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - issues: write - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code Review - id: claude-review - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' - plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - From 28cf93ce4c3739c63e9667b3526bda125062b7db Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Fri, 24 Apr 2026 11:58:30 +0300 Subject: [PATCH 04/12] feat: sync AI extraction export rows on promise status changes Add hooks to PromiseStatus collection to trigger resync of AI extraction export rows when a status is updated or deleted. This ensures data consistency between promise statuses and their associated export rows. Implement syncAIExtractionExportRowsForStatus function that finds all export rows linked to a status, identifies unique AI extractions, and triggers resync for each. Update test to verify deduplication behavior and proper payload calls. Also fix unique key generation to use Payload array row IDs instead of uniqueId to handle duplicate uniqueId values, and remove unnecessary eslint-disable comment. --- src/collections/PromiseStatus.ts | 21 ++++ src/lib/aiExtractionExportRows.ts | 36 ++++++ src/types/airtableSchema.ts | 2 +- tests/int/aiExtractionExportRows.int.spec.ts | 109 ++++++++++++++++++- 4 files changed, 163 insertions(+), 5 deletions(-) diff --git a/src/collections/PromiseStatus.ts b/src/collections/PromiseStatus.ts index e4ae9e37..eec6b743 100644 --- a/src/collections/PromiseStatus.ts +++ b/src/collections/PromiseStatus.ts @@ -1,8 +1,29 @@ +import { syncAIExtractionExportRowsForStatus } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; import { colorPickerField } from "@innovixx/payload-color-picker-field"; export const PromiseStatus: CollectionConfig = { slug: "promise-status", + hooks: { + afterChange: [ + async ({ doc, req }) => { + await syncAIExtractionExportRowsForStatus({ + payload: req.payload, + statusId: String(doc.id), + }); + return doc; + }, + ], + afterDelete: [ + async ({ doc, req }) => { + await syncAIExtractionExportRowsForStatus({ + payload: req.payload, + statusId: String(doc.id), + }); + return doc; + }, + ], + }, admin: { group: { en: "Documents", diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index c24d3eb5..779c249b 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -52,6 +52,7 @@ export type AIExtractionExportRowData = { type ExistingExportRow = { id: string; + aiExtractionId?: string | null; uniqueKey?: string | null; }; @@ -411,6 +412,41 @@ export const syncAIExtractionExportRowsForTenant = async ({ } }; +export const syncAIExtractionExportRowsForStatus = async ({ + payload, + statusId, +}: { + payload: Payload; + statusId: string; +}) => { + const { docs } = await payload.find({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + depth: 0, + limit: 0, + overrideAccess: true, + where: { + statusId: { + equals: statusId, + }, + }, + }); + + const aiExtractionIds = [ + ...new Set( + (docs as ExistingExportRow[]) + .map((doc) => doc.aiExtractionId) + .filter((id): id is string => Boolean(id)), + ), + ]; + + for (const aiExtractionId of aiExtractionIds) { + await syncAIExtractionExportRows({ + aiExtractionId, + payload, + }); + } +}; + export const rebuildAllAIExtractionExportRows = async ({ batchSize = 100, payload, diff --git a/src/types/airtableSchema.ts b/src/types/airtableSchema.ts index 354c9ca8..507326fe 100644 --- a/src/types/airtableSchema.ts +++ b/src/types/airtableSchema.ts @@ -1,5 +1,5 @@ /* DO NOT EDIT: this file was automatically generated by airtable-ts-codegen */ -/* eslint-disable */ + import type { Item, Table } from "airtable-ts"; export interface Countrie extends Item { diff --git a/tests/int/aiExtractionExportRows.int.spec.ts b/tests/int/aiExtractionExportRows.int.spec.ts index c6b039ea..bc94d307 100644 --- a/tests/int/aiExtractionExportRows.int.spec.ts +++ b/tests/int/aiExtractionExportRows.int.spec.ts @@ -1,6 +1,9 @@ -import { buildAIExtractionExportRows } from "@/lib/aiExtractionExportRows"; +import { + buildAIExtractionExportRows, + syncAIExtractionExportRowsForStatus, +} from "@/lib/aiExtractionExportRows"; import type { AiExtraction } from "@/payload-types"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; describe("AI extraction export rows", () => { it("creates one flat export row per extraction entry", () => { @@ -56,7 +59,7 @@ describe("AI extraction export rows", () => { expect(rows).toHaveLength(2); expect(rows[0]).toMatchObject({ - uniqueKey: "ai-extraction-1:unique-1", + uniqueKey: "ai-extraction-1:row-1", tenantId: "tenant-1", tenantName: "Kenya", tenantCountry: "KEN", @@ -66,7 +69,8 @@ describe("AI extraction export rows", () => { documentTitle: "Manifesto PDF", documentUrl: "https://example.com/manifesto.pdf", aiExtractionId: "ai-extraction-1", - extractionRowId: "unique-1", + extractionRowId: "row-1", + uniqueId: "unique-1", category: "Health", summary: "Build new clinics", statusId: "status-1", @@ -77,6 +81,7 @@ describe("AI extraction export rows", () => { }); expect(rows[1]).toMatchObject({ uniqueKey: "ai-extraction-1:row-2", + extractionRowId: "row-2", category: "Education", statusId: "", statusLabel: "", @@ -113,4 +118,100 @@ describe("AI extraction export rows", () => { tenantName: "", }); }); + + it("uses the Payload array row id as the primary export row key", () => { + const extractionDoc = { + id: "ai-extraction-3", + extractions: [ + { + id: "row-1", + uniqueId: "duplicate-id", + category: "Health", + summary: "Row one", + source: "Source one", + }, + { + id: "row-2", + uniqueId: "duplicate-id", + category: "Education", + summary: "Row two", + source: "Source two", + }, + ], + } as AiExtraction; + + const rows = buildAIExtractionExportRows(extractionDoc); + + expect(rows).toHaveLength(2); + expect(rows.map((row) => row.extractionRowId)).toEqual(["row-1", "row-2"]); + expect(rows.map((row) => row.uniqueKey)).toEqual([ + "ai-extraction-3:row-1", + "ai-extraction-3:row-2", + ]); + expect(rows.map((row) => row.uniqueId)).toEqual([ + "duplicate-id", + "duplicate-id", + ]); + }); + + it("resyncs each affected extraction once when a promise status changes", async () => { + const payload = { + create: vi.fn(), + delete: vi.fn(), + find: vi + .fn() + .mockResolvedValueOnce({ + docs: [ + { aiExtractionId: "ai-extraction-1" }, + { aiExtractionId: "ai-extraction-1" }, + { aiExtractionId: "ai-extraction-2" }, + ], + }) + .mockResolvedValueOnce({ docs: [] }) + .mockResolvedValueOnce({ docs: [] }), + findByID: vi + .fn() + .mockResolvedValueOnce({ + id: "ai-extraction-1", + extractions: [], + }) + .mockResolvedValueOnce({ + id: "ai-extraction-2", + extractions: [], + }), + update: vi.fn(), + }; + + await syncAIExtractionExportRowsForStatus({ + payload: payload as never, + statusId: "status-1", + }); + + expect(payload.find).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + collection: "ai-extraction-export-rows", + where: { + statusId: { + equals: "status-1", + }, + }, + }), + ); + expect(payload.findByID).toHaveBeenCalledTimes(2); + expect(payload.findByID).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + collection: "ai-extractions", + id: "ai-extraction-1", + }), + ); + expect(payload.findByID).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + collection: "ai-extractions", + id: "ai-extraction-2", + }), + ); + }); }); From f60fcf16cdf0e1f4005e50de3154a85bae63f4ae Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 10:37:53 +0300 Subject: [PATCH 05/12] perf(export): optimize deletion of AI extraction export rows Replace find-then-delete loop with a single delete operation using where clause to improve performance and reduce database round trips. --- src/lib/aiExtractionExportRows.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 779c249b..6286e8fa 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -108,7 +108,7 @@ const getDocumentUrl = (document: PayloadDocument | null): string => { const getExtractionRowId = ( extraction: ExtractionItem, index: number, -): string => extraction.uniqueId ?? extraction.id ?? String(index + 1); +): string => getId(extraction) || String(index + 1); export const buildAIExtractionExportRows = ( extractionDoc: AiExtraction, @@ -193,21 +193,11 @@ const deleteRowsWhere = async ({ payload: Payload; where: Where; }) => { - const { docs } = await payload.find({ + await payload.delete({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - depth: 0, - limit: 0, overrideAccess: true, where, }); - - for (const row of docs as ExistingExportRow[]) { - await payload.delete({ - collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - id: row.id, - overrideAccess: true, - }); - } }; export const syncAIExtractionExportRows = async ({ From b366672e94bca0212bd74ee88847a75172496313 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:49:08 +0300 Subject: [PATCH 06/12] feat(ai-extraction): add error handling and background job queuing for export row sync - Wrap all export row sync/delete operations in try-catch blocks with proper error logging - Add background job queuing for tenant and political entity sync operations to improve performance - Pass request context through all sync operations for proper access control - Add indexes to frequently queried fields in AIExtractionExportRows collection - Extend sync task to support tenant and political entity scoped operations --- src/collections/AIExtractionExportRows.ts | 17 +++-- src/collections/AIExtractions.ts | 34 +++++++--- src/collections/Documents.ts | 34 +++++++--- src/collections/PoliticalEntities/index.ts | 46 +++++++++---- src/collections/PromiseStatus.ts | 34 +++++++--- src/collections/Tenant/index.ts | 45 +++++++++---- src/lib/aiExtractionExportRows.ts | 78 ++++++++++++++-------- src/tasks/syncAIExtractionExportRows.ts | 72 +++++++++++++++++++- 8 files changed, 279 insertions(+), 81 deletions(-) diff --git a/src/collections/AIExtractionExportRows.ts b/src/collections/AIExtractionExportRows.ts index f33a5c0e..c5a32d9e 100644 --- a/src/collections/AIExtractionExportRows.ts +++ b/src/collections/AIExtractionExportRows.ts @@ -28,9 +28,14 @@ const relationshipField = ({ custom: disabledExportRelationship, }); -const textField = (name: string, label: string): Field => ({ +const textField = ( + name: string, + label: string, + { index = false }: { index?: boolean } = {}, +): Field => ({ name, type: "text", + index, label, admin: { readOnly: true, @@ -97,21 +102,21 @@ export const AIExtractionExportRows: CollectionConfig = { name: "status", relationTo: "promise-status", }), - textField("tenantId", "Tenant ID"), + textField("tenantId", "Tenant ID", { index: true }), textField("tenantName", "Tenant Name"), textField("tenantCountry", "Tenant Country"), textField("tenantLocale", "Tenant Locale"), - textField("politicalEntityId", "Political Entity ID"), + textField("politicalEntityId", "Political Entity ID", { index: true }), textField("politicalEntityName", "Political Entity Name"), textField("politicalEntitySlug", "Political Entity Slug"), textField("politicalEntityPosition", "Political Entity Position"), - textField("documentId", "Document ID"), + textField("documentId", "Document ID", { index: true }), textField("documentTitle", "Document Title"), textField("documentUrl", "Document URL"), textField("documentLanguage", "Document Language"), textField("documentType", "Document Type"), textField("documentAirtableID", "Document Airtable ID"), - textField("aiExtractionId", "AI Extraction ID"), + textField("aiExtractionId", "AI Extraction ID", { index: true }), textField("aiExtractionTitle", "AI Extraction Title"), textField("extractionRowId", "Extraction Row ID"), textField("uniqueId", "Unique ID"), @@ -125,7 +130,7 @@ export const AIExtractionExportRows: CollectionConfig = { readOnly: true, }, }, - textField("statusId", "Status ID"), + textField("statusId", "Status ID", { index: true }), textField("statusLabel", "Status Label"), textField("statusMeedanId", "Status Meedan ID"), textField("checkMediaId", "CheckMedia ID"), diff --git a/src/collections/AIExtractions.ts b/src/collections/AIExtractions.ts index 2979da9a..5108c2e4 100644 --- a/src/collections/AIExtractions.ts +++ b/src/collections/AIExtractions.ts @@ -29,19 +29,37 @@ export const AIExtractions: CollectionConfig = { hooks: { afterChange: [ async ({ doc, req }) => { - await syncAIExtractionExportRows({ - aiExtractionId: String(doc.id), - payload: req.payload, - }); + try { + await syncAIExtractionExportRows({ + aiExtractionId: String(doc.id), + payload: req.payload, + req, + }); + } catch (err) { + req.payload.logger.error({ + aiExtractionId: String(doc.id), + err, + msg: "Failed to sync AI extraction export rows after AI extraction change", + }); + } return doc; }, ], afterDelete: [ async ({ doc, req }) => { - await deleteAIExtractionExportRowsForAIExtraction({ - aiExtractionId: String(doc.id), - payload: req.payload, - }); + try { + await deleteAIExtractionExportRowsForAIExtraction({ + aiExtractionId: String(doc.id), + payload: req.payload, + req, + }); + } catch (err) { + req.payload.logger.error({ + aiExtractionId: String(doc.id), + err, + msg: "Failed to delete AI extraction export rows after AI extraction delete", + }); + } return doc; }, ], diff --git a/src/collections/Documents.ts b/src/collections/Documents.ts index 76d0945d..c61b20af 100644 --- a/src/collections/Documents.ts +++ b/src/collections/Documents.ts @@ -23,19 +23,37 @@ export const Documents: CollectionConfig = { hooks: { afterChange: [ async ({ doc, req }) => { - await syncAIExtractionExportRowsForDocument({ - documentId: String(doc.id), - payload: req.payload, - }); + try { + await syncAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload: req.payload, + req, + }); + } catch (err) { + req.payload.logger.error({ + documentId: String(doc.id), + err, + msg: "Failed to sync AI extraction export rows after document change", + }); + } return doc; }, ], afterDelete: [ async ({ doc, req }) => { - await deleteAIExtractionExportRowsForDocument({ - documentId: String(doc.id), - payload: req.payload, - }); + try { + await deleteAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload: req.payload, + req, + }); + } catch (err) { + req.payload.logger.error({ + documentId: String(doc.id), + err, + msg: "Failed to delete AI extraction export rows after document delete", + }); + } return doc; }, ], diff --git a/src/collections/PoliticalEntities/index.ts b/src/collections/PoliticalEntities/index.ts index 7fc12095..4d37d39d 100644 --- a/src/collections/PoliticalEntities/index.ts +++ b/src/collections/PoliticalEntities/index.ts @@ -1,13 +1,12 @@ import { airtableID } from "@/fields/airtableID"; import { image } from "@/fields/image"; -import { - deleteAIExtractionExportRowsForPoliticalEntity, - syncAIExtractionExportRowsForPoliticalEntity, -} from "@/lib/aiExtractionExportRows"; +import { deleteAIExtractionExportRowsForPoliticalEntity } from "@/lib/aiExtractionExportRows"; import { slugField } from "@/fields/slug"; import { CollectionConfig } from "payload"; import { ensureUniqueSlug } from "./hooks/ensureUniqueSlug"; +const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; + export const PoliticalEntities: CollectionConfig = { slug: "political-entities", labels: { @@ -40,19 +39,42 @@ export const PoliticalEntities: CollectionConfig = { hooks: { afterChange: [ async ({ doc, req }) => { - await syncAIExtractionExportRowsForPoliticalEntity({ - payload: req.payload, - politicalEntityId: String(doc.id), - }); + try { + await req.payload.jobs.queue({ + input: { + politicalEntityId: String(doc.id), + scope: "politicalEntity", + }, + overrideAccess: true, + queue: exportRowSyncQueue, + req, + task: "syncAIExtractionExportRows", + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to queue AI extraction export row sync after political entity change", + politicalEntityId: String(doc.id), + }); + } return doc; }, ], afterDelete: [ async ({ doc, req }) => { - await deleteAIExtractionExportRowsForPoliticalEntity({ - payload: req.payload, - politicalEntityId: String(doc.id), - }); + try { + await deleteAIExtractionExportRowsForPoliticalEntity({ + payload: req.payload, + politicalEntityId: String(doc.id), + req, + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to delete AI extraction export rows after political entity delete", + politicalEntityId: String(doc.id), + }); + } return doc; }, ], diff --git a/src/collections/PromiseStatus.ts b/src/collections/PromiseStatus.ts index eec6b743..c92699aa 100644 --- a/src/collections/PromiseStatus.ts +++ b/src/collections/PromiseStatus.ts @@ -7,19 +7,37 @@ export const PromiseStatus: CollectionConfig = { hooks: { afterChange: [ async ({ doc, req }) => { - await syncAIExtractionExportRowsForStatus({ - payload: req.payload, - statusId: String(doc.id), - }); + try { + await syncAIExtractionExportRowsForStatus({ + payload: req.payload, + req, + statusId: String(doc.id), + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to sync AI extraction export rows after promise status change", + statusId: String(doc.id), + }); + } return doc; }, ], afterDelete: [ async ({ doc, req }) => { - await syncAIExtractionExportRowsForStatus({ - payload: req.payload, - statusId: String(doc.id), - }); + try { + await syncAIExtractionExportRowsForStatus({ + payload: req.payload, + req, + statusId: String(doc.id), + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to sync AI extraction export rows after promise status delete", + statusId: String(doc.id), + }); + } return doc; }, ], diff --git a/src/collections/Tenant/index.ts b/src/collections/Tenant/index.ts index 5a1d02d2..6ce53c59 100644 --- a/src/collections/Tenant/index.ts +++ b/src/collections/Tenant/index.ts @@ -1,12 +1,10 @@ import { countriesByContinent, getCountryFlag } from "@/data/countries"; import { airtableID } from "@/fields/airtableID"; -import { - deleteAIExtractionExportRowsForTenant, - syncAIExtractionExportRowsForTenant, -} from "@/lib/aiExtractionExportRows"; +import { deleteAIExtractionExportRowsForTenant } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; const africanCountries = countriesByContinent("Africa"); +const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; export const Tenants: CollectionConfig = { slug: "tenants", @@ -95,19 +93,42 @@ export const Tenants: CollectionConfig = { hooks: { afterChange: [ async ({ doc, req }) => { - await syncAIExtractionExportRowsForTenant({ - payload: req.payload, - tenantId: String(doc.id), - }); + try { + await req.payload.jobs.queue({ + input: { + scope: "tenant", + tenantId: String(doc.id), + }, + overrideAccess: true, + queue: exportRowSyncQueue, + req, + task: "syncAIExtractionExportRows", + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to queue AI extraction export row sync after tenant change", + tenantId: String(doc.id), + }); + } return doc; }, ], afterDelete: [ async ({ doc, req }) => { - await deleteAIExtractionExportRowsForTenant({ - payload: req.payload, - tenantId: String(doc.id), - }); + try { + await deleteAIExtractionExportRowsForTenant({ + payload: req.payload, + req, + tenantId: String(doc.id), + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to delete AI extraction export rows after tenant delete", + tenantId: String(doc.id), + }); + } return doc; }, ], diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 6286e8fa..123d0e31 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -5,7 +5,7 @@ import type { PromiseStatus, Tenant, } from "@/payload-types"; -import type { Payload, Where } from "payload"; +import type { Payload, PayloadRequest, Where } from "payload"; export const AI_EXTRACTION_EXPORT_ROWS_COLLECTION = "ai-extraction-export-rows" as const; @@ -56,6 +56,11 @@ type ExistingExportRow = { uniqueKey?: string | null; }; +type LocalAPIContext = { + payload: Payload; + req?: Partial; +}; + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -167,15 +172,16 @@ export const buildAIExtractionExportRows = ( const findExistingRows = async ({ aiExtractionId, payload, -}: { + req, +}: LocalAPIContext & { aiExtractionId: string; - payload: Payload; }): Promise => { const { docs } = await payload.find({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, depth: 0, limit: 0, overrideAccess: true, + req, where: { aiExtractionId: { equals: aiExtractionId, @@ -188,14 +194,15 @@ const findExistingRows = async ({ const deleteRowsWhere = async ({ payload, + req, where, -}: { - payload: Payload; +}: LocalAPIContext & { where: Where; }) => { await payload.delete({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, overrideAccess: true, + req, where, }); }; @@ -203,19 +210,20 @@ const deleteRowsWhere = async ({ export const syncAIExtractionExportRows = async ({ aiExtractionId, payload, -}: { + req, +}: LocalAPIContext & { aiExtractionId: string; - payload: Payload; }) => { const extractionDoc = (await payload.findByID({ collection: "ai-extractions", depth: 3, id: aiExtractionId, overrideAccess: true, + req, })) as AiExtraction; const rows = buildAIExtractionExportRows(extractionDoc); - const existingRows = await findExistingRows({ aiExtractionId, payload }); + const existingRows = await findExistingRows({ aiExtractionId, payload, req }); const existingRowsByKey = new Map( existingRows .filter((row) => row.uniqueKey) @@ -233,6 +241,7 @@ export const syncAIExtractionExportRows = async ({ data: row, id: existingRow.id, overrideAccess: true, + req, }); continue; } @@ -241,6 +250,7 @@ export const syncAIExtractionExportRows = async ({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, data: row, overrideAccess: true, + req, }); } @@ -253,6 +263,7 @@ export const syncAIExtractionExportRows = async ({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, id: row.id, overrideAccess: true, + req, }); } }; @@ -260,12 +271,13 @@ export const syncAIExtractionExportRows = async ({ export const deleteAIExtractionExportRowsForAIExtraction = async ({ aiExtractionId, payload, -}: { + req, +}: LocalAPIContext & { aiExtractionId: string; - payload: Payload; }) => deleteRowsWhere({ payload, + req, where: { aiExtractionId: { equals: aiExtractionId, @@ -276,12 +288,13 @@ export const deleteAIExtractionExportRowsForAIExtraction = async ({ export const deleteAIExtractionExportRowsForDocument = async ({ documentId, payload, -}: { + req, +}: LocalAPIContext & { documentId: string; - payload: Payload; }) => deleteRowsWhere({ payload, + req, where: { documentId: { equals: documentId, @@ -292,12 +305,13 @@ export const deleteAIExtractionExportRowsForDocument = async ({ export const deleteAIExtractionExportRowsForPoliticalEntity = async ({ payload, politicalEntityId, -}: { - payload: Payload; + req, +}: LocalAPIContext & { politicalEntityId: string; }) => deleteRowsWhere({ payload, + req, where: { politicalEntityId: { equals: politicalEntityId, @@ -308,12 +322,13 @@ export const deleteAIExtractionExportRowsForPoliticalEntity = async ({ export const deleteAIExtractionExportRowsForTenant = async ({ payload, tenantId, -}: { - payload: Payload; + req, +}: LocalAPIContext & { tenantId: string; }) => deleteRowsWhere({ payload, + req, where: { tenantId: { equals: tenantId, @@ -324,15 +339,16 @@ export const deleteAIExtractionExportRowsForTenant = async ({ export const syncAIExtractionExportRowsForDocument = async ({ documentId, payload, -}: { + req, +}: LocalAPIContext & { documentId: string; - payload: Payload; }) => { const { docs } = await payload.find({ collection: "ai-extractions", depth: 0, limit: 0, overrideAccess: true, + req, where: { document: { equals: documentId, @@ -344,6 +360,7 @@ export const syncAIExtractionExportRowsForDocument = async ({ await syncAIExtractionExportRows({ aiExtractionId: String(doc.id), payload, + req, }); } }; @@ -351,8 +368,8 @@ export const syncAIExtractionExportRowsForDocument = async ({ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ payload, politicalEntityId, -}: { - payload: Payload; + req, +}: LocalAPIContext & { politicalEntityId: string; }) => { const { docs } = await payload.find({ @@ -360,6 +377,7 @@ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ depth: 0, limit: 0, overrideAccess: true, + req, where: { politicalEntity: { equals: politicalEntityId, @@ -371,6 +389,7 @@ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ await syncAIExtractionExportRowsForDocument({ documentId: String(doc.id), payload, + req, }); } }; @@ -378,8 +397,8 @@ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ export const syncAIExtractionExportRowsForTenant = async ({ payload, tenantId, -}: { - payload: Payload; + req, +}: LocalAPIContext & { tenantId: string; }) => { const { docs } = await payload.find({ @@ -387,6 +406,7 @@ export const syncAIExtractionExportRowsForTenant = async ({ depth: 0, limit: 0, overrideAccess: true, + req, where: { tenant: { equals: tenantId, @@ -398,6 +418,7 @@ export const syncAIExtractionExportRowsForTenant = async ({ await syncAIExtractionExportRowsForPoliticalEntity({ payload, politicalEntityId: String(entity.id), + req, }); } }; @@ -405,8 +426,8 @@ export const syncAIExtractionExportRowsForTenant = async ({ export const syncAIExtractionExportRowsForStatus = async ({ payload, statusId, -}: { - payload: Payload; + req, +}: LocalAPIContext & { statusId: string; }) => { const { docs } = await payload.find({ @@ -414,6 +435,7 @@ export const syncAIExtractionExportRowsForStatus = async ({ depth: 0, limit: 0, overrideAccess: true, + req, where: { statusId: { equals: statusId, @@ -433,6 +455,7 @@ export const syncAIExtractionExportRowsForStatus = async ({ await syncAIExtractionExportRows({ aiExtractionId, payload, + req, }); } }; @@ -440,12 +463,13 @@ export const syncAIExtractionExportRowsForStatus = async ({ export const rebuildAllAIExtractionExportRows = async ({ batchSize = 100, payload, -}: { + req, +}: LocalAPIContext & { batchSize?: number; - payload: Payload; }) => { await deleteRowsWhere({ payload, + req, where: {}, }); @@ -460,12 +484,14 @@ export const rebuildAllAIExtractionExportRows = async ({ limit: batchSize, overrideAccess: true, page, + req, }); for (const doc of docs as Pick[]) { await syncAIExtractionExportRows({ aiExtractionId: String(doc.id), payload, + req, }); processed += 1; } diff --git a/src/tasks/syncAIExtractionExportRows.ts b/src/tasks/syncAIExtractionExportRows.ts index 2a63d464..a2d1a140 100644 --- a/src/tasks/syncAIExtractionExportRows.ts +++ b/src/tasks/syncAIExtractionExportRows.ts @@ -1,7 +1,20 @@ -import { rebuildAllAIExtractionExportRows } from "@/lib/aiExtractionExportRows"; +import { + rebuildAllAIExtractionExportRows, + syncAIExtractionExportRowsForPoliticalEntity, + syncAIExtractionExportRowsForTenant, +} from "@/lib/aiExtractionExportRows"; import { TaskConfig } from "payload"; import { getTaskLogger, withTaskTracing } from "./utils"; +type SyncExportRowsInput = { + politicalEntityId?: string; + scope?: "all" | "politicalEntity" | "tenant"; + tenantId?: string; +}; + +const isSyncExportRowsInput = (value: unknown): value is SyncExportRowsInput => + typeof value === "object" && value !== null; + export const SyncAIExtractionExportRows: TaskConfig = { slug: "syncAIExtractionExportRows", label: "Sync AI Extraction Export Rows", @@ -9,6 +22,62 @@ export const SyncAIExtractionExportRows: TaskConfig = { "syncAIExtractionExportRows", async ({ req, input }) => { const logger = getTaskLogger(req, "syncAIExtractionExportRows", input); + const parsedInput = isSyncExportRowsInput(input) ? input : {}; + + if ( + parsedInput.scope === "tenant" && + typeof parsedInput.tenantId === "string" + ) { + logger.info({ + message: + "syncAIExtractionExportRows:: Starting tenant-scoped export row sync", + tenantId: parsedInput.tenantId, + }); + + await syncAIExtractionExportRowsForTenant({ + payload: req.payload, + req, + tenantId: parsedInput.tenantId, + }); + + logger.info({ + message: + "syncAIExtractionExportRows:: Completed tenant-scoped export row sync", + tenantId: parsedInput.tenantId, + }); + + return { output: { scope: "tenant", tenantId: parsedInput.tenantId } }; + } + + if ( + parsedInput.scope === "politicalEntity" && + typeof parsedInput.politicalEntityId === "string" + ) { + logger.info({ + message: + "syncAIExtractionExportRows:: Starting political-entity-scoped export row sync", + politicalEntityId: parsedInput.politicalEntityId, + }); + + await syncAIExtractionExportRowsForPoliticalEntity({ + payload: req.payload, + politicalEntityId: parsedInput.politicalEntityId, + req, + }); + + logger.info({ + message: + "syncAIExtractionExportRows:: Completed political-entity-scoped export row sync", + politicalEntityId: parsedInput.politicalEntityId, + }); + + return { + output: { + politicalEntityId: parsedInput.politicalEntityId, + scope: "politicalEntity", + }, + }; + } logger.info( "syncAIExtractionExportRows:: Rebuilding AI extraction export rows", @@ -16,6 +85,7 @@ export const SyncAIExtractionExportRows: TaskConfig = { const result = await rebuildAllAIExtractionExportRows({ payload: req.payload, + req, }); logger.info({ From 848caca6c2f7d8d6a5a65f4cbf674ff20ee27d5f Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:05:17 +0300 Subject: [PATCH 07/12] refactor(ai-extraction-export): queue sync tasks and improve batching Replace direct sync calls in collection hooks with job queue to defer processing Add batching utility for large collections to prevent memory issues Update rebuild function to delete stale rows instead of clearing all rows --- src/collections/AIExtractions.ts | 20 ++- src/collections/Documents.ts | 20 ++- src/lib/aiExtractionExportRows.ts | 203 +++++++++++++++--------- src/tasks/syncAIExtractionExportRows.ts | 67 +++++++- 4 files changed, 215 insertions(+), 95 deletions(-) diff --git a/src/collections/AIExtractions.ts b/src/collections/AIExtractions.ts index 5108c2e4..2a7f53af 100644 --- a/src/collections/AIExtractions.ts +++ b/src/collections/AIExtractions.ts @@ -1,9 +1,8 @@ -import { - deleteAIExtractionExportRowsForAIExtraction, - syncAIExtractionExportRows, -} from "@/lib/aiExtractionExportRows"; +import { deleteAIExtractionExportRowsForAIExtraction } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; +const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; + export const AIExtractions: CollectionConfig = { slug: "ai-extractions", labels: { @@ -30,16 +29,21 @@ export const AIExtractions: CollectionConfig = { afterChange: [ async ({ doc, req }) => { try { - await syncAIExtractionExportRows({ - aiExtractionId: String(doc.id), - payload: req.payload, + await req.payload.jobs.queue({ + input: { + aiExtractionId: String(doc.id), + scope: "aiExtraction", + }, + overrideAccess: true, + queue: exportRowSyncQueue, req, + task: "syncAIExtractionExportRows", }); } catch (err) { req.payload.logger.error({ aiExtractionId: String(doc.id), err, - msg: "Failed to sync AI extraction export rows after AI extraction change", + msg: "Failed to queue AI extraction export row sync after AI extraction change", }); } return doc; diff --git a/src/collections/Documents.ts b/src/collections/Documents.ts index c61b20af..feed0970 100644 --- a/src/collections/Documents.ts +++ b/src/collections/Documents.ts @@ -1,10 +1,9 @@ import { airtableID } from "@/fields/airtableID"; -import { - deleteAIExtractionExportRowsForDocument, - syncAIExtractionExportRowsForDocument, -} from "@/lib/aiExtractionExportRows"; +import { deleteAIExtractionExportRowsForDocument } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; +const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; + export const Documents: CollectionConfig = { slug: "documents", labels: { @@ -24,16 +23,21 @@ export const Documents: CollectionConfig = { afterChange: [ async ({ doc, req }) => { try { - await syncAIExtractionExportRowsForDocument({ - documentId: String(doc.id), - payload: req.payload, + await req.payload.jobs.queue({ + input: { + documentId: String(doc.id), + scope: "document", + }, + overrideAccess: true, + queue: exportRowSyncQueue, req, + task: "syncAIExtractionExportRows", }); } catch (err) { req.payload.logger.error({ documentId: String(doc.id), err, - msg: "Failed to sync AI extraction export rows after document change", + msg: "Failed to queue AI extraction export row sync after document change", }); } return doc; diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 123d0e31..8262d496 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -10,6 +10,8 @@ import type { Payload, PayloadRequest, Where } from "payload"; export const AI_EXTRACTION_EXPORT_ROWS_COLLECTION = "ai-extraction-export-rows" as const; +const DEFAULT_SYNC_BATCH_SIZE = 100; + type ExtractionItem = NonNullable< NonNullable[number] >; @@ -61,6 +63,12 @@ type LocalAPIContext = { req?: Partial; }; +type SyncCollectionSlug = + | typeof AI_EXTRACTION_EXPORT_ROWS_COLLECTION + | "ai-extractions" + | "documents" + | "political-entities"; + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -115,6 +123,42 @@ const getExtractionRowId = ( index: number, ): string => getId(extraction) || String(index + 1); +const forEachMatchingDoc = async ({ + batchSize = DEFAULT_SYNC_BATCH_SIZE, + collection, + onDoc, + payload, + req, + where, +}: LocalAPIContext & { + batchSize?: number; + collection: SyncCollectionSlug; + onDoc: (doc: T) => Promise | void; + where?: Where; +}) => { + let hasNextPage = true; + let page = 1; + + while (hasNextPage) { + const result = await payload.find({ + collection, + depth: 0, + limit: batchSize, + overrideAccess: true, + page, + req, + where, + }); + + for (const doc of result.docs as T[]) { + await onDoc(doc); + } + + hasNextPage = result.hasNextPage; + page += 1; + } +}; + export const buildAIExtractionExportRows = ( extractionDoc: AiExtraction, ): AIExtractionExportRowData[] => { @@ -176,11 +220,14 @@ const findExistingRows = async ({ }: LocalAPIContext & { aiExtractionId: string; }): Promise => { - const { docs } = await payload.find({ + const docs: ExistingExportRow[] = []; + + await forEachMatchingDoc({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - depth: 0, - limit: 0, - overrideAccess: true, + onDoc: (doc) => { + docs.push(doc); + }, + payload, req, where: { aiExtractionId: { @@ -189,7 +236,7 @@ const findExistingRows = async ({ }, }); - return docs as ExistingExportRow[]; + return docs; }; const deleteRowsWhere = async ({ @@ -343,11 +390,16 @@ export const syncAIExtractionExportRowsForDocument = async ({ }: LocalAPIContext & { documentId: string; }) => { - const { docs } = await payload.find({ + await forEachMatchingDoc>({ collection: "ai-extractions", - depth: 0, - limit: 0, - overrideAccess: true, + onDoc: async (doc) => { + await syncAIExtractionExportRows({ + aiExtractionId: String(doc.id), + payload, + req, + }); + }, + payload, req, where: { document: { @@ -355,14 +407,6 @@ export const syncAIExtractionExportRowsForDocument = async ({ }, }, }); - - for (const doc of docs as Pick[]) { - await syncAIExtractionExportRows({ - aiExtractionId: String(doc.id), - payload, - req, - }); - } }; export const syncAIExtractionExportRowsForPoliticalEntity = async ({ @@ -372,11 +416,16 @@ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ }: LocalAPIContext & { politicalEntityId: string; }) => { - const { docs } = await payload.find({ + await forEachMatchingDoc>({ collection: "documents", - depth: 0, - limit: 0, - overrideAccess: true, + onDoc: async (doc) => { + await syncAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload, + req, + }); + }, + payload, req, where: { politicalEntity: { @@ -384,14 +433,6 @@ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ }, }, }); - - for (const doc of docs as Pick[]) { - await syncAIExtractionExportRowsForDocument({ - documentId: String(doc.id), - payload, - req, - }); - } }; export const syncAIExtractionExportRowsForTenant = async ({ @@ -401,11 +442,16 @@ export const syncAIExtractionExportRowsForTenant = async ({ }: LocalAPIContext & { tenantId: string; }) => { - const { docs } = await payload.find({ + await forEachMatchingDoc>({ collection: "political-entities", - depth: 0, - limit: 0, - overrideAccess: true, + onDoc: async (entity) => { + await syncAIExtractionExportRowsForPoliticalEntity({ + payload, + politicalEntityId: String(entity.id), + req, + }); + }, + payload, req, where: { tenant: { @@ -413,14 +459,6 @@ export const syncAIExtractionExportRowsForTenant = async ({ }, }, }); - - for (const entity of docs as Pick[]) { - await syncAIExtractionExportRowsForPoliticalEntity({ - payload, - politicalEntityId: String(entity.id), - req, - }); - } }; export const syncAIExtractionExportRowsForStatus = async ({ @@ -430,11 +468,16 @@ export const syncAIExtractionExportRowsForStatus = async ({ }: LocalAPIContext & { statusId: string; }) => { - const { docs } = await payload.find({ + const aiExtractionIds = new Set(); + + await forEachMatchingDoc({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - depth: 0, - limit: 0, - overrideAccess: true, + onDoc: (doc) => { + if (doc.aiExtractionId) { + aiExtractionIds.add(doc.aiExtractionId); + } + }, + payload, req, where: { statusId: { @@ -443,14 +486,6 @@ export const syncAIExtractionExportRowsForStatus = async ({ }, }); - const aiExtractionIds = [ - ...new Set( - (docs as ExistingExportRow[]) - .map((doc) => doc.aiExtractionId) - .filter((id): id is string => Boolean(id)), - ), - ]; - for (const aiExtractionId of aiExtractionIds) { await syncAIExtractionExportRows({ aiExtractionId, @@ -461,44 +496,56 @@ export const syncAIExtractionExportRowsForStatus = async ({ }; export const rebuildAllAIExtractionExportRows = async ({ - batchSize = 100, + batchSize = DEFAULT_SYNC_BATCH_SIZE, payload, req, }: LocalAPIContext & { batchSize?: number; }) => { - await deleteRowsWhere({ - payload, - req, - where: {}, - }); - - let hasNextPage = true; - let page = 1; + const syncedAiExtractionIds = new Set(); + const staleRowIds: string[] = []; let processed = 0; - while (hasNextPage) { - const { docs, hasNextPage: nextPage } = await payload.find({ - collection: "ai-extractions", - depth: 0, - limit: batchSize, - overrideAccess: true, - page, - req, - }); - - for (const doc of docs as Pick[]) { + await forEachMatchingDoc>({ + batchSize, + collection: "ai-extractions", + onDoc: async (doc) => { + const aiExtractionId = String(doc.id); + syncedAiExtractionIds.add(aiExtractionId); await syncAIExtractionExportRows({ - aiExtractionId: String(doc.id), + aiExtractionId, payload, req, }); processed += 1; - } + }, + payload, + req, + }); - hasNextPage = nextPage; - page += 1; + await forEachMatchingDoc({ + batchSize, + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + onDoc: (row) => { + if ( + !row.aiExtractionId || + !syncedAiExtractionIds.has(row.aiExtractionId) + ) { + staleRowIds.push(row.id); + } + }, + payload, + req, + }); + + for (const id of staleRowIds) { + await payload.delete({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + id, + overrideAccess: true, + req, + }); } - return { processed }; + return { deletedStaleRows: staleRowIds.length, processed }; }; diff --git a/src/tasks/syncAIExtractionExportRows.ts b/src/tasks/syncAIExtractionExportRows.ts index a2d1a140..bb5ae9c1 100644 --- a/src/tasks/syncAIExtractionExportRows.ts +++ b/src/tasks/syncAIExtractionExportRows.ts @@ -1,5 +1,7 @@ import { rebuildAllAIExtractionExportRows, + syncAIExtractionExportRows, + syncAIExtractionExportRowsForDocument, syncAIExtractionExportRowsForPoliticalEntity, syncAIExtractionExportRowsForTenant, } from "@/lib/aiExtractionExportRows"; @@ -7,8 +9,10 @@ import { TaskConfig } from "payload"; import { getTaskLogger, withTaskTracing } from "./utils"; type SyncExportRowsInput = { + aiExtractionId?: string; + documentId?: string; politicalEntityId?: string; - scope?: "all" | "politicalEntity" | "tenant"; + scope?: "aiExtraction" | "all" | "document" | "politicalEntity" | "tenant"; tenantId?: string; }; @@ -24,6 +28,66 @@ export const SyncAIExtractionExportRows: TaskConfig = { const logger = getTaskLogger(req, "syncAIExtractionExportRows", input); const parsedInput = isSyncExportRowsInput(input) ? input : {}; + if ( + parsedInput.scope === "aiExtraction" && + typeof parsedInput.aiExtractionId === "string" + ) { + logger.info({ + aiExtractionId: parsedInput.aiExtractionId, + message: + "syncAIExtractionExportRows:: Starting AI extraction scoped export row sync", + }); + + await syncAIExtractionExportRows({ + aiExtractionId: parsedInput.aiExtractionId, + payload: req.payload, + req, + }); + + logger.info({ + aiExtractionId: parsedInput.aiExtractionId, + message: + "syncAIExtractionExportRows:: Completed AI extraction scoped export row sync", + }); + + return { + output: { + aiExtractionId: parsedInput.aiExtractionId, + scope: "aiExtraction", + }, + }; + } + + if ( + parsedInput.scope === "document" && + typeof parsedInput.documentId === "string" + ) { + logger.info({ + documentId: parsedInput.documentId, + message: + "syncAIExtractionExportRows:: Starting document scoped export row sync", + }); + + await syncAIExtractionExportRowsForDocument({ + documentId: parsedInput.documentId, + payload: req.payload, + req, + }); + + logger.info({ + documentId: parsedInput.documentId, + message: + "syncAIExtractionExportRows:: Completed document scoped export row sync", + }); + + return { + output: { + documentId: parsedInput.documentId, + scope: "document", + }, + }; + } + if ( parsedInput.scope === "tenant" && typeof parsedInput.tenantId === "string" @@ -89,6 +153,7 @@ export const SyncAIExtractionExportRows: TaskConfig = { }); logger.info({ + deletedStaleRows: result.deletedStaleRows, message: "syncAIExtractionExportRows:: Completed AI extraction export row rebuild", processed: result.processed, From 07949fef04dd932ecd42dc94e86b1f53ea4817c0 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:04:04 +0300 Subject: [PATCH 08/12] refactor(ai-extraction-export): replace direct sync with job queue and optimize deletion - Queue sync tasks for tenant and promise status changes/delete operations instead of direct execution - Add support for status-scoped sync in task handler - Optimize stale row deletion in rebuildAllAIExtractionExportRows to use bulk delete operations - Improve error logging messages to reflect queuing behavior --- src/collections/PromiseStatus.ts | 29 ++++++--- src/collections/Tenant/index.ts | 14 +++-- src/lib/aiExtractionExportRows.ts | 80 +++++++++++++++---------- src/tasks/syncAIExtractionExportRows.ts | 40 ++++++++++++- 4 files changed, 118 insertions(+), 45 deletions(-) diff --git a/src/collections/PromiseStatus.ts b/src/collections/PromiseStatus.ts index c92699aa..7b74985a 100644 --- a/src/collections/PromiseStatus.ts +++ b/src/collections/PromiseStatus.ts @@ -1,22 +1,28 @@ -import { syncAIExtractionExportRowsForStatus } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; import { colorPickerField } from "@innovixx/payload-color-picker-field"; +const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; + export const PromiseStatus: CollectionConfig = { slug: "promise-status", hooks: { afterChange: [ async ({ doc, req }) => { try { - await syncAIExtractionExportRowsForStatus({ - payload: req.payload, + await req.payload.jobs.queue({ + input: { + scope: "status", + statusId: String(doc.id), + }, + overrideAccess: true, + queue: exportRowSyncQueue, req, - statusId: String(doc.id), + task: "syncAIExtractionExportRows", }); } catch (err) { req.payload.logger.error({ err, - msg: "Failed to sync AI extraction export rows after promise status change", + msg: "Failed to queue AI extraction export row sync after promise status change", statusId: String(doc.id), }); } @@ -26,15 +32,20 @@ export const PromiseStatus: CollectionConfig = { afterDelete: [ async ({ doc, req }) => { try { - await syncAIExtractionExportRowsForStatus({ - payload: req.payload, + await req.payload.jobs.queue({ + input: { + scope: "status", + statusId: String(doc.id), + }, + overrideAccess: true, + queue: exportRowSyncQueue, req, - statusId: String(doc.id), + task: "syncAIExtractionExportRows", }); } catch (err) { req.payload.logger.error({ err, - msg: "Failed to sync AI extraction export rows after promise status delete", + msg: "Failed to queue AI extraction export row sync after promise status delete", statusId: String(doc.id), }); } diff --git a/src/collections/Tenant/index.ts b/src/collections/Tenant/index.ts index 6ce53c59..9ce88e26 100644 --- a/src/collections/Tenant/index.ts +++ b/src/collections/Tenant/index.ts @@ -1,6 +1,5 @@ import { countriesByContinent, getCountryFlag } from "@/data/countries"; import { airtableID } from "@/fields/airtableID"; -import { deleteAIExtractionExportRowsForTenant } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; const africanCountries = countriesByContinent("Africa"); @@ -117,15 +116,20 @@ export const Tenants: CollectionConfig = { afterDelete: [ async ({ doc, req }) => { try { - await deleteAIExtractionExportRowsForTenant({ - payload: req.payload, + await req.payload.jobs.queue({ + input: { + scope: "tenant", + tenantId: String(doc.id), + }, + overrideAccess: true, + queue: exportRowSyncQueue, req, - tenantId: String(doc.id), + task: "syncAIExtractionExportRows", }); } catch (err) { req.payload.logger.error({ err, - msg: "Failed to delete AI extraction export rows after tenant delete", + msg: "Failed to queue AI extraction export row sync after tenant delete", tenantId: String(doc.id), }); } diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 8262d496..4929b291 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -301,16 +301,34 @@ export const syncAIExtractionExportRows = async ({ }); } - for (const row of existingRows) { - if (row.uniqueKey && syncedKeys.has(row.uniqueKey)) { - continue; - } - - await payload.delete({ - collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - id: row.id, - overrideAccess: true, + if (syncedKeys.size > 0) { + await deleteRowsWhere({ + payload, req, + where: { + and: [ + { + aiExtractionId: { + equals: aiExtractionId, + }, + }, + { + uniqueKey: { + not_in: [...syncedKeys], + }, + }, + ], + }, + }); + } else { + await deleteRowsWhere({ + payload, + req, + where: { + aiExtractionId: { + equals: aiExtractionId, + }, + }, }); } }; @@ -503,7 +521,6 @@ export const rebuildAllAIExtractionExportRows = async ({ batchSize?: number; }) => { const syncedAiExtractionIds = new Set(); - const staleRowIds: string[] = []; let processed = 0; await forEachMatchingDoc>({ @@ -523,29 +540,32 @@ export const rebuildAllAIExtractionExportRows = async ({ req, }); - await forEachMatchingDoc({ - batchSize, + const deletedStaleRows = await payload.count({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - onDoc: (row) => { - if ( - !row.aiExtractionId || - !syncedAiExtractionIds.has(row.aiExtractionId) - ) { - staleRowIds.push(row.id); - } - }, - payload, + overrideAccess: true, req, + where: + syncedAiExtractionIds.size > 0 + ? { + aiExtractionId: { + not_in: [...syncedAiExtractionIds], + }, + } + : {}, }); - for (const id of staleRowIds) { - await payload.delete({ - collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, - id, - overrideAccess: true, - req, - }); - } + await deleteRowsWhere({ + payload, + req, + where: + syncedAiExtractionIds.size > 0 + ? { + aiExtractionId: { + not_in: [...syncedAiExtractionIds], + }, + } + : {}, + }); - return { deletedStaleRows: staleRowIds.length, processed }; + return { deletedStaleRows: deletedStaleRows.totalDocs, processed }; }; diff --git a/src/tasks/syncAIExtractionExportRows.ts b/src/tasks/syncAIExtractionExportRows.ts index bb5ae9c1..008199ee 100644 --- a/src/tasks/syncAIExtractionExportRows.ts +++ b/src/tasks/syncAIExtractionExportRows.ts @@ -3,6 +3,7 @@ import { syncAIExtractionExportRows, syncAIExtractionExportRowsForDocument, syncAIExtractionExportRowsForPoliticalEntity, + syncAIExtractionExportRowsForStatus, syncAIExtractionExportRowsForTenant, } from "@/lib/aiExtractionExportRows"; import { TaskConfig } from "payload"; @@ -12,7 +13,14 @@ type SyncExportRowsInput = { aiExtractionId?: string; documentId?: string; politicalEntityId?: string; - scope?: "aiExtraction" | "all" | "document" | "politicalEntity" | "tenant"; + scope?: + | "aiExtraction" + | "all" + | "document" + | "politicalEntity" + | "status" + | "tenant"; + statusId?: string; tenantId?: string; }; @@ -88,6 +96,36 @@ export const SyncAIExtractionExportRows: TaskConfig = { }; } + if ( + parsedInput.scope === "status" && + typeof parsedInput.statusId === "string" + ) { + logger.info({ + message: + "syncAIExtractionExportRows:: Starting status scoped export row sync", + statusId: parsedInput.statusId, + }); + + await syncAIExtractionExportRowsForStatus({ + payload: req.payload, + req, + statusId: parsedInput.statusId, + }); + + logger.info({ + message: + "syncAIExtractionExportRows:: Completed status scoped export row sync", + statusId: parsedInput.statusId, + }); + + return { + output: { + scope: "status", + statusId: parsedInput.statusId, + }, + }; + } + if ( parsedInput.scope === "tenant" && typeof parsedInput.tenantId === "string" From 99b7c64170bc458351e696e60e95511f8a676c65 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:27:17 +0300 Subject: [PATCH 09/12] fix(aiExtractionExportRows): prevent accidental deletion of all export rows Add early return when no AI extractions are found during rebuild to avoid unnecessary count and delete operations. Also replace queued job with direct function call for tenant deletion cleanup to ensure immediate removal of associated export rows. --- src/collections/Tenant/index.ts | 14 +++++------- src/lib/aiExtractionExportRows.ts | 4 ++++ tests/int/aiExtractionExportRows.int.spec.ts | 23 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/collections/Tenant/index.ts b/src/collections/Tenant/index.ts index 9ce88e26..6ce53c59 100644 --- a/src/collections/Tenant/index.ts +++ b/src/collections/Tenant/index.ts @@ -1,5 +1,6 @@ import { countriesByContinent, getCountryFlag } from "@/data/countries"; import { airtableID } from "@/fields/airtableID"; +import { deleteAIExtractionExportRowsForTenant } from "@/lib/aiExtractionExportRows"; import { CollectionConfig } from "payload"; const africanCountries = countriesByContinent("Africa"); @@ -116,20 +117,15 @@ export const Tenants: CollectionConfig = { afterDelete: [ async ({ doc, req }) => { try { - await req.payload.jobs.queue({ - input: { - scope: "tenant", - tenantId: String(doc.id), - }, - overrideAccess: true, - queue: exportRowSyncQueue, + await deleteAIExtractionExportRowsForTenant({ + payload: req.payload, req, - task: "syncAIExtractionExportRows", + tenantId: String(doc.id), }); } catch (err) { req.payload.logger.error({ err, - msg: "Failed to queue AI extraction export row sync after tenant delete", + msg: "Failed to delete AI extraction export rows after tenant delete", tenantId: String(doc.id), }); } diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 4929b291..5bc0e929 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -540,6 +540,10 @@ export const rebuildAllAIExtractionExportRows = async ({ req, }); + if (syncedAiExtractionIds.size === 0) { + return { deletedStaleRows: 0, processed }; + } + const deletedStaleRows = await payload.count({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, overrideAccess: true, diff --git a/tests/int/aiExtractionExportRows.int.spec.ts b/tests/int/aiExtractionExportRows.int.spec.ts index bc94d307..89372dcd 100644 --- a/tests/int/aiExtractionExportRows.int.spec.ts +++ b/tests/int/aiExtractionExportRows.int.spec.ts @@ -1,5 +1,6 @@ import { buildAIExtractionExportRows, + rebuildAllAIExtractionExportRows, syncAIExtractionExportRowsForStatus, } from "@/lib/aiExtractionExportRows"; import type { AiExtraction } from "@/payload-types"; @@ -214,4 +215,26 @@ describe("AI extraction export rows", () => { }), ); }); + + it("does not wipe export rows when rebuild finds no AI extractions", async () => { + const payload = { + count: vi.fn(), + create: vi.fn(), + delete: vi.fn(), + find: vi.fn().mockResolvedValueOnce({ + docs: [], + hasNextPage: false, + }), + findByID: vi.fn(), + update: vi.fn(), + }; + + const result = await rebuildAllAIExtractionExportRows({ + payload: payload as never, + }); + + expect(result).toEqual({ deletedStaleRows: 0, processed: 0 }); + expect(payload.count).not.toHaveBeenCalled(); + expect(payload.delete).not.toHaveBeenCalled(); + }); }); From bf4c809dfdfa75e9a74d5c02716fda73d40024e3 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:50:40 +0300 Subject: [PATCH 10/12] refactor: extract AI extraction export row sync logic to shared hooks Move inline job queueing logic from collection configs to dedicated hook files Extract shared type definitions and utility functions to central module Add new PromiseStatus collection with sync hooks Create new Documents and AIExtractions collections with sync hooks --- src/collections/AIExtractions/hooks/index.ts | 46 +++++++++++++ .../index.ts} | 52 ++------------- src/collections/Documents/hooks/index.ts | 46 +++++++++++++ .../{Documents.ts => Documents/index.ts} | 52 ++------------- .../PoliticalEntities/hooks/index.ts | 46 +++++++++++++ src/collections/PoliticalEntities/index.ts | 52 ++------------- src/collections/PromiseStatus/hooks/index.ts | 44 +++++++++++++ .../index.ts} | 57 ++-------------- src/collections/Tenant/hooks/index.ts | 58 +++++++++++++++++ src/collections/Tenant/index.ts | 65 ++++--------------- src/lib/aiExtractionExportRowsJobs.ts | 49 ++++++++++++++ src/tasks/syncAIExtractionExportRows.ts | 20 ++---- 12 files changed, 332 insertions(+), 255 deletions(-) create mode 100644 src/collections/AIExtractions/hooks/index.ts rename src/collections/{AIExtractions.ts => AIExtractions/index.ts} (72%) create mode 100644 src/collections/Documents/hooks/index.ts rename src/collections/{Documents.ts => Documents/index.ts} (72%) create mode 100644 src/collections/PoliticalEntities/hooks/index.ts create mode 100644 src/collections/PromiseStatus/hooks/index.ts rename src/collections/{PromiseStatus.ts => PromiseStatus/index.ts} (58%) create mode 100644 src/collections/Tenant/hooks/index.ts create mode 100644 src/lib/aiExtractionExportRowsJobs.ts diff --git a/src/collections/AIExtractions/hooks/index.ts b/src/collections/AIExtractions/hooks/index.ts new file mode 100644 index 00000000..eb70fd29 --- /dev/null +++ b/src/collections/AIExtractions/hooks/index.ts @@ -0,0 +1,46 @@ +import { deleteAIExtractionExportRowsForAIExtraction } from "@/lib/aiExtractionExportRows"; +import { queueAIExtractionExportRowsSync } from "@/lib/aiExtractionExportRowsJobs"; +import type { AiExtraction } from "@/payload-types"; +import type { + CollectionAfterChangeHook, + CollectionAfterDeleteHook, +} from "payload"; + +export const queueAIExtractionExportRowsSyncAfterAIExtractionChange: CollectionAfterChangeHook< + AiExtraction +> = async ({ doc, req }) => { + await queueAIExtractionExportRowsSync({ + errorMessage: + "Failed to queue AI extraction export row sync after AI extraction change", + input: { + aiExtractionId: String(doc.id), + scope: "aiExtraction", + }, + logContext: { + aiExtractionId: String(doc.id), + }, + req, + }); + + return doc; +}; + +export const deleteAIExtractionExportRowsAfterAIExtractionDelete: CollectionAfterDeleteHook< + AiExtraction +> = async ({ doc, req }) => { + try { + await deleteAIExtractionExportRowsForAIExtraction({ + aiExtractionId: String(doc.id), + payload: req.payload, + req, + }); + } catch (err) { + req.payload.logger.error({ + aiExtractionId: String(doc.id), + err, + msg: "Failed to delete AI extraction export rows after AI extraction delete", + }); + } + + return doc; +}; diff --git a/src/collections/AIExtractions.ts b/src/collections/AIExtractions/index.ts similarity index 72% rename from src/collections/AIExtractions.ts rename to src/collections/AIExtractions/index.ts index 2a7f53af..8b7af557 100644 --- a/src/collections/AIExtractions.ts +++ b/src/collections/AIExtractions/index.ts @@ -1,7 +1,8 @@ -import { deleteAIExtractionExportRowsForAIExtraction } from "@/lib/aiExtractionExportRows"; -import { CollectionConfig } from "payload"; - -const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; +import type { CollectionConfig } from "payload"; +import { + deleteAIExtractionExportRowsAfterAIExtractionDelete, + queueAIExtractionExportRowsSyncAfterAIExtractionChange, +} from "./hooks"; export const AIExtractions: CollectionConfig = { slug: "ai-extractions", @@ -26,47 +27,8 @@ export const AIExtractions: CollectionConfig = { read: () => true, }, hooks: { - afterChange: [ - async ({ doc, req }) => { - try { - await req.payload.jobs.queue({ - input: { - aiExtractionId: String(doc.id), - scope: "aiExtraction", - }, - overrideAccess: true, - queue: exportRowSyncQueue, - req, - task: "syncAIExtractionExportRows", - }); - } catch (err) { - req.payload.logger.error({ - aiExtractionId: String(doc.id), - err, - msg: "Failed to queue AI extraction export row sync after AI extraction change", - }); - } - return doc; - }, - ], - afterDelete: [ - async ({ doc, req }) => { - try { - await deleteAIExtractionExportRowsForAIExtraction({ - aiExtractionId: String(doc.id), - payload: req.payload, - req, - }); - } catch (err) { - req.payload.logger.error({ - aiExtractionId: String(doc.id), - err, - msg: "Failed to delete AI extraction export rows after AI extraction delete", - }); - } - return doc; - }, - ], + afterChange: [queueAIExtractionExportRowsSyncAfterAIExtractionChange], + afterDelete: [deleteAIExtractionExportRowsAfterAIExtractionDelete], }, fields: [ { diff --git a/src/collections/Documents/hooks/index.ts b/src/collections/Documents/hooks/index.ts new file mode 100644 index 00000000..accc1bf6 --- /dev/null +++ b/src/collections/Documents/hooks/index.ts @@ -0,0 +1,46 @@ +import { deleteAIExtractionExportRowsForDocument } from "@/lib/aiExtractionExportRows"; +import { queueAIExtractionExportRowsSync } from "@/lib/aiExtractionExportRowsJobs"; +import type { Document as PayloadDocument } from "@/payload-types"; +import type { + CollectionAfterChangeHook, + CollectionAfterDeleteHook, +} from "payload"; + +export const queueAIExtractionExportRowsSyncAfterDocumentChange: CollectionAfterChangeHook< + PayloadDocument +> = async ({ doc, req }) => { + await queueAIExtractionExportRowsSync({ + errorMessage: + "Failed to queue AI extraction export row sync after document change", + input: { + documentId: String(doc.id), + scope: "document", + }, + logContext: { + documentId: String(doc.id), + }, + req, + }); + + return doc; +}; + +export const deleteAIExtractionExportRowsAfterDocumentDelete: CollectionAfterDeleteHook< + PayloadDocument +> = async ({ doc, req }) => { + try { + await deleteAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload: req.payload, + req, + }); + } catch (err) { + req.payload.logger.error({ + documentId: String(doc.id), + err, + msg: "Failed to delete AI extraction export rows after document delete", + }); + } + + return doc; +}; diff --git a/src/collections/Documents.ts b/src/collections/Documents/index.ts similarity index 72% rename from src/collections/Documents.ts rename to src/collections/Documents/index.ts index feed0970..a967cc81 100644 --- a/src/collections/Documents.ts +++ b/src/collections/Documents/index.ts @@ -1,8 +1,9 @@ +import type { CollectionConfig } from "payload"; import { airtableID } from "@/fields/airtableID"; -import { deleteAIExtractionExportRowsForDocument } from "@/lib/aiExtractionExportRows"; -import { CollectionConfig } from "payload"; - -const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; +import { + deleteAIExtractionExportRowsAfterDocumentDelete, + queueAIExtractionExportRowsSyncAfterDocumentChange, +} from "./hooks"; export const Documents: CollectionConfig = { slug: "documents", @@ -20,47 +21,8 @@ export const Documents: CollectionConfig = { read: () => true, }, hooks: { - afterChange: [ - async ({ doc, req }) => { - try { - await req.payload.jobs.queue({ - input: { - documentId: String(doc.id), - scope: "document", - }, - overrideAccess: true, - queue: exportRowSyncQueue, - req, - task: "syncAIExtractionExportRows", - }); - } catch (err) { - req.payload.logger.error({ - documentId: String(doc.id), - err, - msg: "Failed to queue AI extraction export row sync after document change", - }); - } - return doc; - }, - ], - afterDelete: [ - async ({ doc, req }) => { - try { - await deleteAIExtractionExportRowsForDocument({ - documentId: String(doc.id), - payload: req.payload, - req, - }); - } catch (err) { - req.payload.logger.error({ - documentId: String(doc.id), - err, - msg: "Failed to delete AI extraction export rows after document delete", - }); - } - return doc; - }, - ], + afterChange: [queueAIExtractionExportRowsSyncAfterDocumentChange], + afterDelete: [deleteAIExtractionExportRowsAfterDocumentDelete], }, admin: { defaultColumns: ["title", "politicalEntity", "language", "type"], diff --git a/src/collections/PoliticalEntities/hooks/index.ts b/src/collections/PoliticalEntities/hooks/index.ts new file mode 100644 index 00000000..175749ce --- /dev/null +++ b/src/collections/PoliticalEntities/hooks/index.ts @@ -0,0 +1,46 @@ +import { deleteAIExtractionExportRowsForPoliticalEntity } from "@/lib/aiExtractionExportRows"; +import { queueAIExtractionExportRowsSync } from "@/lib/aiExtractionExportRowsJobs"; +import type { PoliticalEntity } from "@/payload-types"; +import type { + CollectionAfterChangeHook, + CollectionAfterDeleteHook, +} from "payload"; + +export const queueAIExtractionExportRowsSyncAfterPoliticalEntityChange: CollectionAfterChangeHook< + PoliticalEntity +> = async ({ doc, req }) => { + await queueAIExtractionExportRowsSync({ + errorMessage: + "Failed to queue AI extraction export row sync after political entity change", + input: { + politicalEntityId: String(doc.id), + scope: "politicalEntity", + }, + logContext: { + politicalEntityId: String(doc.id), + }, + req, + }); + + return doc; +}; + +export const deleteAIExtractionExportRowsAfterPoliticalEntityDelete: CollectionAfterDeleteHook< + PoliticalEntity +> = async ({ doc, req }) => { + try { + await deleteAIExtractionExportRowsForPoliticalEntity({ + payload: req.payload, + politicalEntityId: String(doc.id), + req, + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to delete AI extraction export rows after political entity delete", + politicalEntityId: String(doc.id), + }); + } + + return doc; +}; diff --git a/src/collections/PoliticalEntities/index.ts b/src/collections/PoliticalEntities/index.ts index 4d37d39d..ccf0c01e 100644 --- a/src/collections/PoliticalEntities/index.ts +++ b/src/collections/PoliticalEntities/index.ts @@ -1,12 +1,13 @@ import { airtableID } from "@/fields/airtableID"; import { image } from "@/fields/image"; -import { deleteAIExtractionExportRowsForPoliticalEntity } from "@/lib/aiExtractionExportRows"; import { slugField } from "@/fields/slug"; -import { CollectionConfig } from "payload"; +import type { CollectionConfig } from "payload"; +import { + deleteAIExtractionExportRowsAfterPoliticalEntityDelete, + queueAIExtractionExportRowsSyncAfterPoliticalEntityChange, +} from "./hooks"; import { ensureUniqueSlug } from "./hooks/ensureUniqueSlug"; -const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; - export const PoliticalEntities: CollectionConfig = { slug: "political-entities", labels: { @@ -37,47 +38,8 @@ export const PoliticalEntities: CollectionConfig = { ], }, hooks: { - afterChange: [ - async ({ doc, req }) => { - try { - await req.payload.jobs.queue({ - input: { - politicalEntityId: String(doc.id), - scope: "politicalEntity", - }, - overrideAccess: true, - queue: exportRowSyncQueue, - req, - task: "syncAIExtractionExportRows", - }); - } catch (err) { - req.payload.logger.error({ - err, - msg: "Failed to queue AI extraction export row sync after political entity change", - politicalEntityId: String(doc.id), - }); - } - return doc; - }, - ], - afterDelete: [ - async ({ doc, req }) => { - try { - await deleteAIExtractionExportRowsForPoliticalEntity({ - payload: req.payload, - politicalEntityId: String(doc.id), - req, - }); - } catch (err) { - req.payload.logger.error({ - err, - msg: "Failed to delete AI extraction export rows after political entity delete", - politicalEntityId: String(doc.id), - }); - } - return doc; - }, - ], + afterChange: [queueAIExtractionExportRowsSyncAfterPoliticalEntityChange], + afterDelete: [deleteAIExtractionExportRowsAfterPoliticalEntityDelete], }, fields: [ { diff --git a/src/collections/PromiseStatus/hooks/index.ts b/src/collections/PromiseStatus/hooks/index.ts new file mode 100644 index 00000000..6a174228 --- /dev/null +++ b/src/collections/PromiseStatus/hooks/index.ts @@ -0,0 +1,44 @@ +import { queueAIExtractionExportRowsSync } from "@/lib/aiExtractionExportRowsJobs"; +import type { PromiseStatus as PromiseStatusDoc } from "@/payload-types"; +import type { + CollectionAfterChangeHook, + CollectionAfterDeleteHook, +} from "payload"; + +export const queueAIExtractionExportRowsSyncAfterPromiseStatusChange: CollectionAfterChangeHook< + PromiseStatusDoc +> = async ({ doc, req }) => { + await queueAIExtractionExportRowsSync({ + errorMessage: + "Failed to queue AI extraction export row sync after promise status change", + input: { + scope: "status", + statusId: String(doc.id), + }, + logContext: { + statusId: String(doc.id), + }, + req, + }); + + return doc; +}; + +export const queueAIExtractionExportRowsSyncAfterPromiseStatusDelete: CollectionAfterDeleteHook< + PromiseStatusDoc +> = async ({ doc, req }) => { + await queueAIExtractionExportRowsSync({ + errorMessage: + "Failed to queue AI extraction export row sync after promise status delete", + input: { + scope: "status", + statusId: String(doc.id), + }, + logContext: { + statusId: String(doc.id), + }, + req, + }); + + return doc; +}; diff --git a/src/collections/PromiseStatus.ts b/src/collections/PromiseStatus/index.ts similarity index 58% rename from src/collections/PromiseStatus.ts rename to src/collections/PromiseStatus/index.ts index 7b74985a..f4d11b43 100644 --- a/src/collections/PromiseStatus.ts +++ b/src/collections/PromiseStatus/index.ts @@ -1,57 +1,15 @@ -import { CollectionConfig } from "payload"; +import type { CollectionConfig } from "payload"; import { colorPickerField } from "@innovixx/payload-color-picker-field"; - -const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; +import { + queueAIExtractionExportRowsSyncAfterPromiseStatusChange, + queueAIExtractionExportRowsSyncAfterPromiseStatusDelete, +} from "./hooks"; export const PromiseStatus: CollectionConfig = { slug: "promise-status", hooks: { - afterChange: [ - async ({ doc, req }) => { - try { - await req.payload.jobs.queue({ - input: { - scope: "status", - statusId: String(doc.id), - }, - overrideAccess: true, - queue: exportRowSyncQueue, - req, - task: "syncAIExtractionExportRows", - }); - } catch (err) { - req.payload.logger.error({ - err, - msg: "Failed to queue AI extraction export row sync after promise status change", - statusId: String(doc.id), - }); - } - return doc; - }, - ], - afterDelete: [ - async ({ doc, req }) => { - try { - await req.payload.jobs.queue({ - input: { - scope: "status", - statusId: String(doc.id), - }, - overrideAccess: true, - queue: exportRowSyncQueue, - req, - task: "syncAIExtractionExportRows", - }); - } catch (err) { - req.payload.logger.error({ - err, - msg: "Failed to queue AI extraction export row sync after promise status delete", - statusId: String(doc.id), - }); - } - return doc; - }, - ], + afterChange: [queueAIExtractionExportRowsSyncAfterPromiseStatusChange], + afterDelete: [queueAIExtractionExportRowsSyncAfterPromiseStatusDelete], }, admin: { group: { @@ -101,7 +59,6 @@ export const PromiseStatus: CollectionConfig = { }, ], }, - { name: "description", type: "textarea", diff --git a/src/collections/Tenant/hooks/index.ts b/src/collections/Tenant/hooks/index.ts new file mode 100644 index 00000000..18cbe2db --- /dev/null +++ b/src/collections/Tenant/hooks/index.ts @@ -0,0 +1,58 @@ +import { countriesByContinent, getCountryFlag } from "@/data/countries"; +import { deleteAIExtractionExportRowsForTenant } from "@/lib/aiExtractionExportRows"; +import { queueAIExtractionExportRowsSync } from "@/lib/aiExtractionExportRowsJobs"; +import type { Tenant } from "@/payload-types"; +import type { + CollectionAfterChangeHook, + CollectionAfterDeleteHook, + CollectionAfterReadHook, +} from "payload"; + +export const queueAIExtractionExportRowsSyncAfterTenantChange: CollectionAfterChangeHook< + Tenant +> = async ({ doc, req }) => { + await queueAIExtractionExportRowsSync({ + errorMessage: + "Failed to queue AI extraction export row sync after tenant change", + input: { + scope: "tenant", + tenantId: String(doc.id), + }, + logContext: { + tenantId: String(doc.id), + }, + req, + }); + + return doc; +}; + +export const deleteAIExtractionExportRowsAfterTenantDelete: CollectionAfterDeleteHook< + Tenant +> = async ({ doc, req }) => { + try { + await deleteAIExtractionExportRowsForTenant({ + payload: req.payload, + req, + tenantId: String(doc.id), + }); + } catch (err) { + req.payload.logger.error({ + err, + msg: "Failed to delete AI extraction export rows after tenant delete", + tenantId: String(doc.id), + }); + } + + return doc; +}; + +export const populateTenantFlagAfterRead: CollectionAfterReadHook< + Tenant +> = async ({ doc }) => { + const tenantWithFlag = doc as Tenant & { flag?: string | null }; + tenantWithFlag.flag = getCountryFlag(doc.country); + return tenantWithFlag; +}; + +export const TENANT_COUNTRY_OPTIONS = countriesByContinent("Africa"); diff --git a/src/collections/Tenant/index.ts b/src/collections/Tenant/index.ts index 6ce53c59..bc67520a 100644 --- a/src/collections/Tenant/index.ts +++ b/src/collections/Tenant/index.ts @@ -1,10 +1,11 @@ -import { countriesByContinent, getCountryFlag } from "@/data/countries"; +import type { CollectionConfig } from "payload"; import { airtableID } from "@/fields/airtableID"; -import { deleteAIExtractionExportRowsForTenant } from "@/lib/aiExtractionExportRows"; -import { CollectionConfig } from "payload"; - -const africanCountries = countriesByContinent("Africa"); -const exportRowSyncQueue = process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; +import { + deleteAIExtractionExportRowsAfterTenantDelete, + populateTenantFlagAfterRead, + queueAIExtractionExportRowsSyncAfterTenantChange, + TENANT_COUNTRY_OPTIONS, +} from "./hooks"; export const Tenants: CollectionConfig = { slug: "tenants", @@ -67,7 +68,7 @@ export const Tenants: CollectionConfig = { { name: "country", type: "select", - options: africanCountries, + options: TENANT_COUNTRY_OPTIONS, unique: true, required: true, label: { @@ -91,52 +92,8 @@ export const Tenants: CollectionConfig = { airtableID(), ], hooks: { - afterChange: [ - async ({ doc, req }) => { - try { - await req.payload.jobs.queue({ - input: { - scope: "tenant", - tenantId: String(doc.id), - }, - overrideAccess: true, - queue: exportRowSyncQueue, - req, - task: "syncAIExtractionExportRows", - }); - } catch (err) { - req.payload.logger.error({ - err, - msg: "Failed to queue AI extraction export row sync after tenant change", - tenantId: String(doc.id), - }); - } - return doc; - }, - ], - afterDelete: [ - async ({ doc, req }) => { - try { - await deleteAIExtractionExportRowsForTenant({ - payload: req.payload, - req, - tenantId: String(doc.id), - }); - } catch (err) { - req.payload.logger.error({ - err, - msg: "Failed to delete AI extraction export rows after tenant delete", - tenantId: String(doc.id), - }); - } - return doc; - }, - ], - afterRead: [ - async ({ doc }) => { - doc.flag = getCountryFlag(doc.country); - return doc; - }, - ], + afterChange: [queueAIExtractionExportRowsSyncAfterTenantChange], + afterDelete: [deleteAIExtractionExportRowsAfterTenantDelete], + afterRead: [populateTenantFlagAfterRead], }, }; diff --git a/src/lib/aiExtractionExportRowsJobs.ts b/src/lib/aiExtractionExportRowsJobs.ts new file mode 100644 index 00000000..26a54299 --- /dev/null +++ b/src/lib/aiExtractionExportRowsJobs.ts @@ -0,0 +1,49 @@ +import type { PayloadRequest } from "payload"; + +export const AI_EXTRACTION_EXPORT_ROWS_SYNC_QUEUE = + process.env.PAYLOAD_JOBS_QUEUE || "everyMinute"; + +export type SyncAIExtractionExportRowsInput = { + aiExtractionId?: string; + documentId?: string; + politicalEntityId?: string; + scope?: + | "aiExtraction" + | "all" + | "document" + | "politicalEntity" + | "status" + | "tenant"; + statusId?: string; + tenantId?: string; +}; + +type QueueAIExtractionExportRowsSyncArgs = { + errorMessage: string; + input: SyncAIExtractionExportRowsInput; + logContext: Record; + req: PayloadRequest; +}; + +export const queueAIExtractionExportRowsSync = async ({ + errorMessage, + input, + logContext, + req, +}: QueueAIExtractionExportRowsSyncArgs): Promise => { + try { + await req.payload.jobs.queue({ + input, + overrideAccess: true, + queue: AI_EXTRACTION_EXPORT_ROWS_SYNC_QUEUE, + req, + task: "syncAIExtractionExportRows", + }); + } catch (err) { + req.payload.logger.error({ + ...logContext, + err, + msg: errorMessage, + }); + } +}; diff --git a/src/tasks/syncAIExtractionExportRows.ts b/src/tasks/syncAIExtractionExportRows.ts index 008199ee..4445af2e 100644 --- a/src/tasks/syncAIExtractionExportRows.ts +++ b/src/tasks/syncAIExtractionExportRows.ts @@ -1,3 +1,4 @@ +import type { SyncAIExtractionExportRowsInput } from "@/lib/aiExtractionExportRowsJobs"; import { rebuildAllAIExtractionExportRows, syncAIExtractionExportRows, @@ -9,22 +10,9 @@ import { import { TaskConfig } from "payload"; import { getTaskLogger, withTaskTracing } from "./utils"; -type SyncExportRowsInput = { - aiExtractionId?: string; - documentId?: string; - politicalEntityId?: string; - scope?: - | "aiExtraction" - | "all" - | "document" - | "politicalEntity" - | "status" - | "tenant"; - statusId?: string; - tenantId?: string; -}; - -const isSyncExportRowsInput = (value: unknown): value is SyncExportRowsInput => +const isSyncExportRowsInput = ( + value: unknown, +): value is SyncAIExtractionExportRowsInput => typeof value === "object" && value !== null; export const SyncAIExtractionExportRows: TaskConfig = { From 1eb30896aac9ec43fb561acc390485f4d0a49286 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:01:40 +0300 Subject: [PATCH 11/12] fix: validate scope and id in syncAIExtractionExportRows task Add input validation to ensure scoped sync operations provide the required identifier. Refuse execution when scope is specified but the corresponding ID is missing or when an unsupported scope is provided. Simplify conditional logic in rebuildAllAIExtractionExportRows by removing unnecessary ternary operators, as the condition always evaluates to true in practice. --- src/lib/aiExtractionExportRows.ts | 26 +++++-------- src/tasks/syncAIExtractionExportRows.ts | 41 ++++++++++++++++++++ tests/int/aiExtractionExportRows.int.spec.ts | 1 + 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 5bc0e929..0885cfa8 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -548,27 +548,21 @@ export const rebuildAllAIExtractionExportRows = async ({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, overrideAccess: true, req, - where: - syncedAiExtractionIds.size > 0 - ? { - aiExtractionId: { - not_in: [...syncedAiExtractionIds], - }, - } - : {}, + where: { + aiExtractionId: { + not_in: [...syncedAiExtractionIds], + }, + }, }); await deleteRowsWhere({ payload, req, - where: - syncedAiExtractionIds.size > 0 - ? { - aiExtractionId: { - not_in: [...syncedAiExtractionIds], - }, - } - : {}, + where: { + aiExtractionId: { + not_in: [...syncedAiExtractionIds], + }, + }, }); return { deletedStaleRows: deletedStaleRows.totalDocs, processed }; diff --git a/src/tasks/syncAIExtractionExportRows.ts b/src/tasks/syncAIExtractionExportRows.ts index 4445af2e..314ea71e 100644 --- a/src/tasks/syncAIExtractionExportRows.ts +++ b/src/tasks/syncAIExtractionExportRows.ts @@ -15,6 +15,17 @@ const isSyncExportRowsInput = ( ): value is SyncAIExtractionExportRowsInput => typeof value === "object" && value !== null; +const scopedInputIsMissingRequiredId = ( + input: SyncAIExtractionExportRowsInput, +): boolean => + (input.scope === "aiExtraction" && + typeof input.aiExtractionId !== "string") || + (input.scope === "document" && typeof input.documentId !== "string") || + (input.scope === "politicalEntity" && + typeof input.politicalEntityId !== "string") || + (input.scope === "status" && typeof input.statusId !== "string") || + (input.scope === "tenant" && typeof input.tenantId !== "string"); + export const SyncAIExtractionExportRows: TaskConfig = { slug: "syncAIExtractionExportRows", label: "Sync AI Extraction Export Rows", @@ -24,6 +35,21 @@ export const SyncAIExtractionExportRows: TaskConfig = { const logger = getTaskLogger(req, "syncAIExtractionExportRows", input); const parsedInput = isSyncExportRowsInput(input) ? input : {}; + if (scopedInputIsMissingRequiredId(parsedInput)) { + logger.error({ + input: parsedInput, + message: + "syncAIExtractionExportRows:: Refusing scoped export row sync with missing identifier", + }); + + return { + output: { + error: "missingIdentifier", + scope: parsedInput.scope, + }, + }; + } + if ( parsedInput.scope === "aiExtraction" && typeof parsedInput.aiExtractionId === "string" @@ -169,6 +195,21 @@ export const SyncAIExtractionExportRows: TaskConfig = { }; } + if (parsedInput.scope && parsedInput.scope !== "all") { + logger.error({ + input: parsedInput, + message: + "syncAIExtractionExportRows:: Refusing export row sync with unsupported scope", + }); + + return { + output: { + error: "unsupportedScope", + scope: parsedInput.scope, + }, + }; + } + logger.info( "syncAIExtractionExportRows:: Rebuilding AI extraction export rows", ); diff --git a/tests/int/aiExtractionExportRows.int.spec.ts b/tests/int/aiExtractionExportRows.int.spec.ts index 89372dcd..23515d32 100644 --- a/tests/int/aiExtractionExportRows.int.spec.ts +++ b/tests/int/aiExtractionExportRows.int.spec.ts @@ -167,6 +167,7 @@ describe("AI extraction export rows", () => { { aiExtractionId: "ai-extraction-1" }, { aiExtractionId: "ai-extraction-2" }, ], + hasNextPage: false, }) .mockResolvedValueOnce({ docs: [] }) .mockResolvedValueOnce({ docs: [] }), From 34e8c21c39c725c63a51dcbc41eaecee4da618b9 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:13:40 +0300 Subject: [PATCH 12/12] fix(aiExtractionExportRows): prevent deletion of concurrently created export rows Improve the rebuildAllAIExtractionExportRows function to only delete export rows for AI extractions that are confirmed missing, rather than using a simple not_in check against the synced IDs. This prevents race conditions where export rows created after the rebuild scan started would be incorrectly deleted. Add helper functions aiExtractionExists and findOrphanedAIExtractionIds to verify each candidate AI extraction still exists in the database before marking its export rows for deletion. Also add error logging wrappers to all sync functions to improve debuggability. --- src/lib/aiExtractionExportRows.ts | 181 ++++++++++++++++--- tests/int/aiExtractionExportRows.int.spec.ts | 64 +++++++ 2 files changed, 223 insertions(+), 22 deletions(-) diff --git a/src/lib/aiExtractionExportRows.ts b/src/lib/aiExtractionExportRows.ts index 0885cfa8..dd202c43 100644 --- a/src/lib/aiExtractionExportRows.ts +++ b/src/lib/aiExtractionExportRows.ts @@ -254,6 +254,91 @@ const deleteRowsWhere = async ({ }); }; +const logScopedSyncError = ({ + context, + err, + message, + payload, +}: { + context: Record; + err: unknown; + message: string; + payload: Payload; +}) => { + payload.logger?.error({ + ...context, + err, + msg: message, + }); +}; + +const aiExtractionExists = async ({ + aiExtractionId, + payload, + req, +}: LocalAPIContext & { + aiExtractionId: string; +}): Promise => { + const result = await payload.find({ + collection: "ai-extractions", + depth: 0, + limit: 1, + overrideAccess: true, + req, + where: { + id: { + equals: aiExtractionId, + }, + }, + }); + + return result.docs.length > 0; +}; + +const findOrphanedAIExtractionIds = async ({ + payload, + req, + retainedAIExtractionIds, +}: LocalAPIContext & { + retainedAIExtractionIds: Set; +}): Promise => { + const candidateAIExtractionIds = new Set(); + const orphanedAIExtractionIds: string[] = []; + + await forEachMatchingDoc({ + collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, + onDoc: (doc) => { + if (doc.aiExtractionId) { + candidateAIExtractionIds.add(doc.aiExtractionId); + } + }, + payload, + req, + where: { + aiExtractionId: { + not_in: [...retainedAIExtractionIds], + }, + }, + }); + + for (const aiExtractionId of candidateAIExtractionIds) { + if ( + retainedAIExtractionIds.has(aiExtractionId) || + (await aiExtractionExists({ + aiExtractionId, + payload, + req, + })) + ) { + continue; + } + + orphanedAIExtractionIds.push(aiExtractionId); + } + + return orphanedAIExtractionIds; +}; + export const syncAIExtractionExportRows = async ({ aiExtractionId, payload, @@ -411,11 +496,21 @@ export const syncAIExtractionExportRowsForDocument = async ({ await forEachMatchingDoc>({ collection: "ai-extractions", onDoc: async (doc) => { - await syncAIExtractionExportRows({ - aiExtractionId: String(doc.id), - payload, - req, - }); + try { + await syncAIExtractionExportRows({ + aiExtractionId: String(doc.id), + payload, + req, + }); + } catch (err) { + logScopedSyncError({ + context: { aiExtractionId: String(doc.id), documentId }, + err, + message: + "Failed to sync AI extraction export rows for document-scoped sync item", + payload, + }); + } }, payload, req, @@ -437,11 +532,21 @@ export const syncAIExtractionExportRowsForPoliticalEntity = async ({ await forEachMatchingDoc>({ collection: "documents", onDoc: async (doc) => { - await syncAIExtractionExportRowsForDocument({ - documentId: String(doc.id), - payload, - req, - }); + try { + await syncAIExtractionExportRowsForDocument({ + documentId: String(doc.id), + payload, + req, + }); + } catch (err) { + logScopedSyncError({ + context: { documentId: String(doc.id), politicalEntityId }, + err, + message: + "Failed to sync AI extraction export rows for political-entity-scoped document", + payload, + }); + } }, payload, req, @@ -463,11 +568,21 @@ export const syncAIExtractionExportRowsForTenant = async ({ await forEachMatchingDoc>({ collection: "political-entities", onDoc: async (entity) => { - await syncAIExtractionExportRowsForPoliticalEntity({ - payload, - politicalEntityId: String(entity.id), - req, - }); + try { + await syncAIExtractionExportRowsForPoliticalEntity({ + payload, + politicalEntityId: String(entity.id), + req, + }); + } catch (err) { + logScopedSyncError({ + context: { politicalEntityId: String(entity.id), tenantId }, + err, + message: + "Failed to sync AI extraction export rows for tenant-scoped political entity", + payload, + }); + } }, payload, req, @@ -505,11 +620,21 @@ export const syncAIExtractionExportRowsForStatus = async ({ }); for (const aiExtractionId of aiExtractionIds) { - await syncAIExtractionExportRows({ - aiExtractionId, - payload, - req, - }); + try { + await syncAIExtractionExportRows({ + aiExtractionId, + payload, + req, + }); + } catch (err) { + logScopedSyncError({ + context: { aiExtractionId, statusId }, + err, + message: + "Failed to sync AI extraction export rows for status-scoped AI extraction", + payload, + }); + } } }; @@ -544,13 +669,25 @@ export const rebuildAllAIExtractionExportRows = async ({ return { deletedStaleRows: 0, processed }; } + // Only delete rows whose parent AI extraction is confirmed missing now. + // This avoids deleting rows created concurrently after the rebuild scan started. + const orphanedAIExtractionIds = await findOrphanedAIExtractionIds({ + payload, + req, + retainedAIExtractionIds: syncedAiExtractionIds, + }); + + if (orphanedAIExtractionIds.length === 0) { + return { deletedStaleRows: 0, processed }; + } + const deletedStaleRows = await payload.count({ collection: AI_EXTRACTION_EXPORT_ROWS_COLLECTION, overrideAccess: true, req, where: { aiExtractionId: { - not_in: [...syncedAiExtractionIds], + in: orphanedAIExtractionIds, }, }, }); @@ -560,7 +697,7 @@ export const rebuildAllAIExtractionExportRows = async ({ req, where: { aiExtractionId: { - not_in: [...syncedAiExtractionIds], + in: orphanedAIExtractionIds, }, }, }); diff --git a/tests/int/aiExtractionExportRows.int.spec.ts b/tests/int/aiExtractionExportRows.int.spec.ts index 23515d32..c19a307b 100644 --- a/tests/int/aiExtractionExportRows.int.spec.ts +++ b/tests/int/aiExtractionExportRows.int.spec.ts @@ -238,4 +238,68 @@ describe("AI extraction export rows", () => { expect(payload.count).not.toHaveBeenCalled(); expect(payload.delete).not.toHaveBeenCalled(); }); + + it("only deletes export rows for AI extractions that no longer exist", async () => { + const payload = { + count: vi.fn().mockResolvedValue({ totalDocs: 2 }), + create: vi.fn(), + delete: vi.fn(), + find: vi + .fn() + .mockResolvedValueOnce({ + docs: [{ id: "ai-extraction-1" }], + hasNextPage: false, + }) + .mockResolvedValueOnce({ + docs: [], + hasNextPage: false, + }) + .mockResolvedValueOnce({ + docs: [ + { aiExtractionId: "ai-extraction-2" }, + { aiExtractionId: "ai-extraction-3" }, + ], + hasNextPage: false, + }) + .mockResolvedValueOnce({ + docs: [], + hasNextPage: false, + }) + .mockResolvedValueOnce({ + docs: [{ id: "ai-extraction-3" }], + hasNextPage: false, + }), + findByID: vi.fn().mockResolvedValue({ + id: "ai-extraction-1", + extractions: [], + }), + update: vi.fn(), + }; + + const result = await rebuildAllAIExtractionExportRows({ + payload: payload as never, + }); + + expect(result).toEqual({ deletedStaleRows: 2, processed: 1 }); + expect(payload.count).toHaveBeenCalledWith( + expect.objectContaining({ + collection: "ai-extraction-export-rows", + where: { + aiExtractionId: { + in: ["ai-extraction-2"], + }, + }, + }), + ); + expect(payload.delete).toHaveBeenCalledWith( + expect.objectContaining({ + collection: "ai-extraction-export-rows", + where: { + aiExtractionId: { + in: ["ai-extraction-2"], + }, + }, + }), + ); + }); });