diff --git a/.github/workflows/onshape_baserow_delta.yml b/.github/workflows/onshape_supabase_delta.yml similarity index 65% rename from .github/workflows/onshape_baserow_delta.yml rename to .github/workflows/onshape_supabase_delta.yml index 8140fe1..6b95e8e 100644 --- a/.github/workflows/onshape_baserow_delta.yml +++ b/.github/workflows/onshape_supabase_delta.yml @@ -1,10 +1,10 @@ -name: Sync Onshape Delta BOM to Baserow +name: Sync Onshape Delta BOM to Supabase on: workflow_dispatch: inputs: dry_run: - description: Resolve the released BOM without calling Baserow + description: Resolve the released BOM without calling Supabase required: false default: false type: boolean @@ -13,16 +13,17 @@ on: required: false type: string repository_dispatch: - types: [bom_sync_delta_baserow] + types: [bom_sync_delta_supabase] concurrency: group: >- - onshape-baserow-delta-${{ github.event_name == 'workflow_dispatch' && + onshape-supabase-delta-${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && 'dry-run' || 'production' }} cancel-in-progress: true jobs: - sync: + sync-supabase: + name: Sync engineering BOM to Supabase if: >- ${{ github.event_name != 'workflow_dispatch' || (!inputs.dry_run && github.ref_name == github.event.repository.default_branch) }} @@ -40,12 +41,12 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: pre-2027-onshape_ci/requirements.txt + cache-dependency-path: pre-2027-onshape_ci/requirements-sync.txt - name: Install dependencies - run: pip install -r pre-2027-onshape_ci/requirements.txt + run: pip install -r pre-2027-onshape_ci/requirements-sync.txt - - name: Sync BOM to Baserow + - name: Sync BOM to Supabase env: ONSHAPE_ACCESS_KEY: ${{ secrets.ONSHAPE_ACCESS_KEY }} ONSHAPE_SECRET_KEY: ${{ secrets.ONSHAPE_SECRET_KEY }} @@ -54,17 +55,10 @@ jobs: inputs.onshape_doc_url || secrets.ONSHAPE_DOC_URL_DELTA }} PARTNUMBER_PREFIXES: "P-190B-26" - BASEROW_API_URL: ${{ vars.BASEROW_API_URL || 'https://api.baserow.io/api' }} - BASEROW_TOKEN: ${{ secrets.BASEROW_TOKEN }} - BASEROW_SYNC_RUNS_TABLE_ID: "1119639" - BASEROW_PARTS_TABLE_ID: "1119641" - BASEROW_REQUIREMENTS_TABLE_ID: "1119642" - BASEROW_OPERATIONS_TABLE_ID: "1169282" - BASEROW_FINISHING_TABLE_ID: "1170619" - BASEROW_STORAGE_LOCATIONS_TABLE_ID: "1119643" - BASEROW_ASSEMBLIES_TABLE_ID: "1119645" + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} + SUPABASE_SECRET_KEY: ${{ secrets.SUPABASE_SECRET_KEY }} GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: python pre-2027-onshape_ci/OnshapeToBaserow.py + run: python pre-2027-onshape_ci/OnshapeToSupabase.py dry-run: if: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} @@ -82,24 +76,24 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: pre-2027-onshape_ci/requirements.txt + cache-dependency-path: pre-2027-onshape_ci/requirements-sync.txt - name: Install dependencies - run: pip install -r pre-2027-onshape_ci/requirements.txt + run: pip install -r pre-2027-onshape_ci/requirements-sync.txt - - name: Resolve released BOM without Baserow + - name: Resolve released BOM without Supabase env: ONSHAPE_ACCESS_KEY: ${{ secrets.ONSHAPE_ACCESS_KEY }} ONSHAPE_SECRET_KEY: ${{ secrets.ONSHAPE_SECRET_KEY }} ONSHAPE_DOC_URL: ${{ inputs.onshape_doc_url || secrets.ONSHAPE_DOC_URL_DELTA }} PARTNUMBER_PREFIXES: "P-190B-26" run: >- - python pre-2027-onshape_ci/OnshapeToBaserow.py + python pre-2027-onshape_ci/OnshapeToSupabase.py --dry-run - --output-json onshape-baserow-dry-run.json + --output-json onshape-supabase-dry-run.json - name: Upload dry-run records uses: actions/upload-artifact@v4 with: - name: onshape-baserow-delta-dry-run - path: onshape-baserow-dry-run.json + name: onshape-supabase-delta-dry-run + path: onshape-supabase-dry-run.json diff --git a/.github/workflows/onshape_baserow_poot_horse.yml b/.github/workflows/onshape_supabase_poot_horse.yml similarity index 77% rename from .github/workflows/onshape_baserow_poot_horse.yml rename to .github/workflows/onshape_supabase_poot_horse.yml index b54ec27..c4bea7d 100644 --- a/.github/workflows/onshape_baserow_poot_horse.yml +++ b/.github/workflows/onshape_supabase_poot_horse.yml @@ -1,10 +1,10 @@ -name: Sync Onshape poot_horse BOM to Baserow +name: Sync Onshape poot_horse BOM to Supabase on: workflow_dispatch: inputs: dry_run: - description: Resolve the released BOM without calling Baserow + description: Resolve the released BOM without calling Supabase required: false default: false type: boolean @@ -27,11 +27,11 @@ on: default: true type: boolean repository_dispatch: - types: [bom_sync_poot_horse_baserow] + types: [bom_sync_poot_horse_supabase] concurrency: group: >- - onshape-baserow-poot_horse-${{ github.event_name == 'workflow_dispatch' && + onshape-supabase-poot_horse-${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && 'dry-run' || 'production' }} cancel-in-progress: true @@ -46,7 +46,8 @@ env: https://frc190.onshape.com/documents/e0e5730aacdbb8063c5716e8/w/bbf4b4765261ae88cc30b198/e/81e8a72bfca21fa77a8e2aea jobs: - sync: + sync-supabase: + name: Sync engineering BOM to Supabase if: >- ${{ github.event_name != 'workflow_dispatch' || (!inputs.dry_run && github.ref_name == github.event.repository.default_branch) }} @@ -64,15 +65,15 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: pre-2027-onshape_ci/requirements.txt + cache-dependency-path: pre-2027-onshape_ci/requirements-sync.txt - name: Install dependencies - run: pip install -r pre-2027-onshape_ci/requirements.txt + run: pip install -r pre-2027-onshape_ci/requirements-sync.txt - - name: Test Onshape to Baserow sync - run: python -m unittest pre-2027-onshape_ci/test_onshape_to_baserow.py + - name: Test Onshape to Supabase sync + run: python -m unittest pre-2027-onshape_ci/test_onshape_to_supabase.py - - name: Sync BOM to Baserow + - name: Sync BOM to Supabase env: ONSHAPE_ACCESS_KEY: ${{ secrets.ONSHAPE_ACCESS_KEY }} ONSHAPE_SECRET_KEY: ${{ secrets.ONSHAPE_SECRET_KEY }} @@ -88,18 +89,11 @@ jobs: SYNC_CAD_FILES: >- ${{ github.event_name != 'workflow_dispatch' || inputs.sync_cad_files }} - BASEROW_API_URL: ${{ vars.BASEROW_API_URL || 'https://api.baserow.io/api' }} - BASEROW_TOKEN: ${{ secrets.BASEROW_TOKEN }} - BASEROW_SYNC_RUNS_TABLE_ID: "1119639" - BASEROW_PARTS_TABLE_ID: "1119641" - BASEROW_REQUIREMENTS_TABLE_ID: "1119642" - BASEROW_OPERATIONS_TABLE_ID: "1169282" - BASEROW_FINISHING_TABLE_ID: "1170619" - BASEROW_STORAGE_LOCATIONS_TABLE_ID: "1119643" - BASEROW_ASSEMBLIES_TABLE_ID: "1119645" + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} + SUPABASE_SECRET_KEY: ${{ secrets.SUPABASE_SECRET_KEY }} ONSHAPE_EXPORT_TIMEOUT_SECONDS: "600" GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: python pre-2027-onshape_ci/OnshapeToBaserow.py + run: python pre-2027-onshape_ci/OnshapeToSupabase.py dry-run: if: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} @@ -117,15 +111,15 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: pre-2027-onshape_ci/requirements.txt + cache-dependency-path: pre-2027-onshape_ci/requirements-sync.txt - name: Install dependencies - run: pip install -r pre-2027-onshape_ci/requirements.txt + run: pip install -r pre-2027-onshape_ci/requirements-sync.txt - - name: Test Onshape to Baserow sync - run: python -m unittest pre-2027-onshape_ci/test_onshape_to_baserow.py + - name: Test Onshape to Supabase sync + run: python -m unittest pre-2027-onshape_ci/test_onshape_to_supabase.py - - name: Resolve released BOM without Baserow + - name: Resolve released BOM without Supabase env: ONSHAPE_ACCESS_KEY: ${{ secrets.ONSHAPE_ACCESS_KEY }} ONSHAPE_SECRET_KEY: ${{ secrets.ONSHAPE_SECRET_KEY }} @@ -136,12 +130,12 @@ jobs: ${{ inputs.subassembly_urls || env.DEFAULT_ONSHAPE_SUBASSEMBLY_URLS }} SYNC_CAD_FILES: ${{ inputs.sync_cad_files }} run: >- - python pre-2027-onshape_ci/OnshapeToBaserow.py + python pre-2027-onshape_ci/OnshapeToSupabase.py --dry-run - --output-json onshape-baserow-dry-run.json + --output-json onshape-supabase-dry-run.json - name: Upload dry-run records uses: actions/upload-artifact@v4 with: - name: onshape-baserow-poot-horse-dry-run - path: onshape-baserow-dry-run.json + name: onshape-supabase-poot-horse-dry-run + path: onshape-supabase-dry-run.json diff --git a/.github/workflows/onshape_supabase_tests.yml b/.github/workflows/onshape_supabase_tests.yml new file mode 100644 index 0000000..625e7bd --- /dev/null +++ b/.github/workflows/onshape_supabase_tests.yml @@ -0,0 +1,34 @@ +name: Test Onshape Supabase engineering sync + +on: + pull_request: + paths: + - 'pre-2027-onshape_ci/**' + - 'supabase/**' + - 'tests/**' + - '.github/workflows/onshape_supabase*' + workflow_dispatch: + +permissions: + contents: read + +jobs: + offline-tests: + name: Mocked Onshape and isolated PostgreSQL tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Test resolution, dry runs and Supabase transport without live APIs + run: python -m unittest discover -s pre-2027-onshape_ci -p test_onshape_to_supabase.py + - uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Install isolated PostgreSQL runtime + run: npm install --no-save --package-lock=false @electric-sql/pglite@0.3.14 + - name: Test engineering transaction against contract fixtures + run: node tests/engineering-sync.test.mjs + - name: Test imported duplicates and missing revision markers + run: node tests/imported-sync.test.mjs diff --git a/.github/workflows/pre-2027-onshape-ci-workflows/onshape_baserow_delta.yml b/.github/workflows/pre-2027-onshape-ci-workflows/onshape_supabase_delta.yml similarity index 57% rename from .github/workflows/pre-2027-onshape-ci-workflows/onshape_baserow_delta.yml rename to .github/workflows/pre-2027-onshape-ci-workflows/onshape_supabase_delta.yml index f93d927..6274a97 100644 --- a/.github/workflows/pre-2027-onshape-ci-workflows/onshape_baserow_delta.yml +++ b/.github/workflows/pre-2027-onshape-ci-workflows/onshape_supabase_delta.yml @@ -1,18 +1,19 @@ -name: Sync Onshape Delta BOM to Baserow +name: Sync Onshape Delta BOM to Supabase on: workflow_dispatch: {} repository_dispatch: - types: [bom_sync_delta_baserow] + types: [bom_sync_delta_supabase] schedule: - cron: "7,22,37,52 * * * *" concurrency: - group: onshape-baserow-delta + group: onshape-supabase-delta cancel-in-progress: true jobs: - sync: + sync-supabase: + name: Sync engineering BOM to Supabase runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -27,24 +28,19 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: pre-2027-onshape_ci/requirements.txt + cache-dependency-path: pre-2027-onshape_ci/requirements-sync.txt - name: Install dependencies - run: pip install -r pre-2027-onshape_ci/requirements.txt + run: pip install -r pre-2027-onshape_ci/requirements-sync.txt - - name: Sync BOM to Baserow + - name: Sync BOM to Supabase env: ONSHAPE_ACCESS_KEY: ${{ secrets.ONSHAPE_ACCESS_KEY }} ONSHAPE_SECRET_KEY: ${{ secrets.ONSHAPE_SECRET_KEY }} ONSHAPE_DOC_URL: ${{ secrets.ONSHAPE_DOC_URL_DELTA }} PARTNUMBER_PREFIXES: "P-190B-26" - BASEROW_API_URL: ${{ vars.BASEROW_API_URL || 'https://api.baserow.io/api' }} - BASEROW_TOKEN: ${{ secrets.BASEROW_TOKEN }} - BASEROW_SYNC_RUNS_TABLE_ID: "1119639" - BASEROW_PARTS_TABLE_ID: "1119641" - BASEROW_REQUIREMENTS_TABLE_ID: "1119642" - BASEROW_STORAGE_LOCATIONS_TABLE_ID: "1119643" - BASEROW_ASSEMBLIES_TABLE_ID: "1119645" + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} + SUPABASE_SECRET_KEY: ${{ secrets.SUPABASE_SECRET_KEY }} GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: python pre-2027-onshape_ci/OnshapeToBaserow.py + run: python pre-2027-onshape_ci/OnshapeToSupabase.py diff --git a/.gitignore b/.gitignore index 8271b98..755a134 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ *.json +/.local-test/ +/.venv/ +node_modules/ +__pycache__/ +*.pyc +.env +.env.* /.idea/.gitignore /.idea/FRC190_github_root.iml /.idea/misc.xml diff --git a/README.md b/README.md index 78c535e..fa35f68 100644 --- a/README.md +++ b/README.md @@ -3,53 +3,50 @@ This is Team 190's org-wide `.github` repository. It provides: - **Org profile** — [`profile/README.md`](profile/README.md), shown on [github.com/Team-190](https://github.com/Team-190). -- **Org-wide default templates** — [`PULL_REQUEST_TEMPLATE.md`](PULL_REQUEST_TEMPLATE.md) and [issue templates](.github/ISSUE_TEMPLATE), used automatically by any repo in the org that doesn't define its own. For code of conduct, contributing guidelines, security reporting, and support, see the [190 Software Knowledge Base](https://team-190.github.io/190-Software-Knowledge-Base/category/software-engineering-practices). -- **Shared CI** — app integrations and general GitHub workflows (Onshape → Baserow BOM sync, GompeiLib sync) under [`.github/workflows`](.github/workflows). +- **Org-wide default templates** — [`PULL_REQUEST_TEMPLATE.md`](PULL_REQUEST_TEMPLATE.md) and [issue templates](.github/ISSUE_TEMPLATE), used automatically by repositories that do not define their own. See the [190 Software Knowledge Base](https://team-190.github.io/190-Software-Knowledge-Base/category/software-engineering-practices) for contributing and security practices. +- **Shared CI** — Onshape → Supabase BOM sync and GompeiLib workflows under [`.github/workflows`](.github/workflows). -## Pre-merge Onshape BOM dry runs +## Onshape engineering BOM sync -An implementation branch can use the repository's existing Onshape Actions -secrets without receiving any Baserow credentials: +The sync resolves released manufacturing roots from either the configured URL +list or direct children of a Main workspace. Only changed roots fetch released +BOMs and drawing revisions. Engineering records and attachment catalogs are +committed through one transactional RPC; shop workflow, assignments, QC, +locations and production quantities remain shop-owned. -```text -gh workflow run onshape_baserow_delta.yml --ref -f dry_run=true -gh workflow run onshape_baserow_poot_horse.yml --ref -f dry_run=true -``` - -Each command checks out the selected branch, reads the unreleased master -workspace to discover direct child assemblies, resolves each child's latest -released BOM, and uploads the resulting JSON. The isolated dry-run job has no Baserow -URL, token, or table IDs. Manual production jobs can run only from the default -branch, while scheduled production syncs are unchanged. +See [Supabase setup and behavior](pre-2027-onshape_ci/SUPABASE_SETUP.md) for the +unapplied migration, required secrets, ownership rules, failure handling and +offline tests. No workflow applies database migrations. -The master workspace is discovery input only and does not define the production -BOM. Each direct child's assembly number and source document are used to call -the Onshape latest-revision API. The child revision's immutable -document/version/element/configuration coordinates define its manufacturing -baseline. +## Pre-merge dry runs -For example, another master workspace can be tested without changing the -poot_horse production secret: +After approval to call Onshape, an implementation branch can use the existing +Onshape Actions secrets without receiving any Supabase credentials: ```text -gh workflow run onshape_baserow_poot_horse.yml \ - --ref \ - -f dry_run=true \ - -f onshape_doc_url="https://frc190.onshape.com/documents/.../w/.../e/..." +gh workflow run onshape_supabase_poot_horse.yml --ref -f dry_run=true +gh workflow run onshape_supabase_delta.yml --ref -f dry_run=true ``` -The URL override is consumed as the discovery master by manual workflow runs, -including production runs from the default branch. If omitted, the workflow -uses its existing `ONSHAPE_DOC_URL_*` Main-workspace secret. Scheduled and -repository-dispatch production runs always use that secret. Because -`dry_run=false` writes to Baserow, validate the same override with a dry run -before using it for production. +Poot Horse defaults to the configured manufacturing-root URL list. Set +`use_subassembly_list=false` to discover direct child assemblies from Main. +Delta uses Main discovery. The master is discovery input, not the production +baseline: each child's latest immutable released version supplies its BOM. -After validation, a default-branch production override can be dispatched with: +The dry-run job reads Onshape and uploads a JSON artifact to GitHub. It receives +no Supabase URL or secret, never generates a missing Onshape BOM, and never +starts a CAD translation or uploads to Storage. An uncached master BOM must be +generated separately before a read-only dry run can inspect it. + +For a different Poot Horse discovery master, after approval: ```text -gh workflow run onshape_baserow_poot_horse.yml \ - --ref main \ - -f dry_run=false \ - -f onshape_doc_url="https://frc190.onshape.com/documents/.../w/.../e/..." +gh workflow run onshape_supabase_poot_horse.yml --ref -f dry_run=true -f use_subassembly_list=false -f onshape_doc_url="https://frc190.onshape.com/documents/.../w/.../e/..." ``` + +Manual production dispatches remain restricted to the default branch. Production +runs require the migration and `NEXT_PUBLIC_SUPABASE_URL` and +`SUPABASE_SECRET_KEY` GitHub Actions secrets. Repository dispatch types are +`bom_sync_poot_horse_supabase` and `bom_sync_delta_supabase`; external dispatchers +must be updated separately. The archived workflow under +`pre-2027-onshape-ci-workflows` remains archived. diff --git a/pre-2027-onshape_ci/BASEROW_SETUP.md b/pre-2027-onshape_ci/BASEROW_SETUP.md deleted file mode 100644 index f2c7ac4..0000000 --- a/pre-2027-onshape_ci/BASEROW_SETUP.md +++ /dev/null @@ -1,341 +0,0 @@ -# Onshape to Baserow setup - -The sync writes only engineering-owned fields. It updates the four `Machine OP` -fields from released Onshape routing metadata, but never updates manufacturing -status, machinist, finishing, current location, QC outcome, or disposition on -an existing production requirement. -On the Operations table it updates the released Onshape routing fields and -manages only the `Planned`/`Ready` operation queue gate; machinist, -in-progress/exception/completion status, instructions, timestamps, and notes -remain manufacturing-owned. - -The poot-horse workflow defaults to `USE_SUBASSEMBLY_LIST=true` and reads the -manufacturing-root URLs stored in `DEFAULT_ONSHAPE_SUBASSEMBLY_URLS` in the -workflow file. Each URL is resolved to its latest released assembly revision, -and every resolved root gets an Assemblies row even when it has no matching -production parts. A manual `subassembly_urls` input can replace the configured -list with one or more comma- or newline-separated URLs. - -Production runs compare those resolved root revisions with the Assemblies table -before fetching any released-root BOM. If every revision is already current, -the run stops there without reading part metadata or drawings, exporting files, -or reading and writing the remaining Baserow tables. A new or changed root runs -the complete synchronization. Main-discovery mode also runs the complete sync -when a child is removed from or re-added to Main. Dry runs intentionally bypass -this shortcut so they continue to produce a complete validation artifact. - -If one configured URL cannot be resolved, list mode logs a warning containing -the exact URL and error, skips that root, and continues syncing the others. The -skipped root's existing Baserow requirements are left unchanged. If none of the -configured roots can be resolved, the run fails before changing Baserow. - -When `USE_SUBASSEMBLY_LIST=false`, and in the Delta workflow, -`ONSHAPE_DOC_URL` points to the master assembly tab in Main. The master does not -need to be released. Its workspace BOM is used only to discover direct `A-...` -child assemblies; nested assemblies are intentionally ignored so their parts -are not counted twice. - -For every discovered direct child, the sync uses the child row's source -document and `A-...` Part Number to resolve its latest released assembly -revision (`et=1`). The assembly Name remains descriptive subsystem metadata; for -example, Name `A-26C-0001` may have released Part Number `A-190B-261132`. -Legacy rows with an `A-...` Name and blank or `N/A` Part Number are still -recognized. Production requirements are built only from that child's immutable -released document/version/element/configuration coordinates. A child without a -release is skipped with a warning, and its existing Baserow requirements are -left unchanged. If no released direct children can be resolved, the run fails -before changing Baserow. - -## Required tables - -The Delta workflow is configured for these Baserow tables: - -| Table | ID | -|---|---:| -| Sync Runs | 1119639 | -| Parts | 1119641 | -| Production Requirements | 1119642 | -| Operations | 1169282 | -| Storage Locations | 1119643 | -| Assemblies | 1119645 | - -Field names are API contracts and must match the names below exactly. - -### Assemblies - -- `Assembly Number` — primary text -- `Subsystem Name` — text -- `Active` — boolean -- `Latest Released Revision` — text -- `Master Baseline Revision` — text -- `Integration Status` — single select or text; include - `Discovered — Master Unreleased` and `Missing from Main — Review` (the legacy - comparison values `Not Compared`, `Not in Master`, `Current in Master`, and - `Newer Revision Available` may remain) -- `Discovery Master` — URL or text; Main-workspace master used to discover this - manufacturing root -- `Onshape Source` — URL; immutable released assembly link -- `Last Synced At` — date with time -- `Sync Schema Version` — text; internal migration marker used to force one - complete sync after managed schema changes -- `Notes` — long text - -### Parts - -- `Part Number` — primary text -- `Name` — text -- `Description` — long text -- `Material` — text -- `Manufacturing Method` — single select or text -- `Vendor` — text -- `Revision` — text -- `OnShape Text` — text -- `Category` — text -- `Onshape Drawing` — URL; immutable released-version drawing link -- `Drawing PDF` — file; exported PDF for the matching released drawing -- `STEP File` — file; AP242 STEP export(s) for the configured part -- `Drawing PDF Export Key` — text; internal cache key -- `STEP Export Key` — text; internal cache key -- `Active` — boolean -- `Last Synced At` — date with time - -PDF and STEP uploads use the filename produced by the applicable Onshape export -rule. The sync does not replace that name with the part number. STEP files are -only exported for parts whose `Manufacturing Method` is `Haas CNC`, -`Shop Sabre CNC`, `Bambu 3D Printer`, `Markforged 3D Printer`, `FormLabs SLA`, -or `FormLabs SLS`. Existing STEP attachments for other methods are preserved, -but the sync does not start or download new STEP translations for them. - -If `Manufacturing Method` is a single select, create choices for every Onshape -value before the first sync, including `SELECT VALUE:` during migration. - -### Production Requirements - -Before running this sync version, a Baserow administrator must manually create -a text field named exactly `Source Document` on Production Requirements table -`1119642`. The sync does not create or modify Baserow schema. If the field is -absent, compatibility filtering omits it from outgoing records. - -- `Production Key` — primary text -- `Part` — link to one Parts row -- `Assembly` — link to one Assemblies row -- `Source Root` — text; independently synchronized manufacturing root -- `Source Assembly Revision` — text; immutable released root revision -- `Required Part Revision` — text; part revision captured by that root release -- `Configuration` — long text -- `Required Quantity` — number -- `BOM Positions` — long text -- `Onshape Source` — URL -- `Source Document` — text; actual Onshape document name containing the part -- `Drawing` — lookup of `Onshape Drawing` through `Part` -- `Drawing PDF` — lookup of `Drawing PDF` through `Part` -- `STEP File` — lookup of `STEP File` through `Part` -- `Machine OP1` — single select; direct value of `Manufacturing Method` -- `Machine OP2` — single select; direct value of `Manufacturing Method OP2` -- `Machine OP3` — single select; direct value of `Manufacturing Method OP3` -- `Machine OP4` — single select; direct value of `Manufacturing Method OP4` -- `Status` — single select; default `Needs Drawing` -- `Machine` — single select -- `Machinist` — text -- `Finishing` — single select with exactly `None`, `Red`, and `Black`; synced - from the released Onshape `Powder Coat Color` property -- `Current Location` — link to one Storage Locations row -- `QC Outcome` — single select; default `Not Inspected` -- `Disposition` — single select; default `Make` -- `Active in BOM` — boolean -- `Engineering Changed` — boolean -- `Last Synced At` — date with time - -Status choices: - -1. Needs Drawing -2. Ready for CAM -3. Ready for Manufacturing -4. On Machine -5. Ready for QC -6. Needs Rework -7. Ready for Finishing -8. Complete - -### Finishing - -This is the active powder-coating work queue. The default grid view filters -`Active` to selected so it shows every current Production Requirement whose -released `Powder Coat Color` is `Red` or `Black`. - -- `Production Key` — primary text -- `Production Requirement` — link to one Production Requirements row -- `Active` — boolean -- `Powder Coat Color` — single select with exactly `Red` and `Black` -- `Required Quantity` — number -- `Last Synced At` — date with time -- `Machinist` — text; assigned manually and preserved by the sync - -Production Requirements with `Powder Coat Color = None` are excluded. Queue -rows are retained but marked inactive when a later release no longer needs -powder coating. `Required Quantity` is informational: finishing does not track -claimed or partially completed quantities. Completing a finishing row means the -Machinist finished the entire Production Requirement quantity. - -### Operations - -- `Operation` — primary text; generated as `|OP1` through `OP4` -- `Production Requirement` — link to Production Requirements; keep the related - field enabled and allow multiple relationships so one requirement can own - multiple operation rows -- `Operation Number` — text or single select with exactly `OP1`, `OP2`, `OP3`, - and `OP4` -- `Machine` — single select or text -- `Status` — single select; default `Planned` -- `Machinist` — text or collaborator -- `Work Instructions` — long text -- `Started At` — date with time -- `Completed At` — date with time -- `Notes` — long text -- `Active in Routing` — boolean; required so removed Onshape operations can be - retained for history but hidden from active queues - -Status choices: - -1. Planned -2. Ready -3. In Progress -4. Blocked -5. Needs Rework -6. Complete - -The released Onshape properties map to the fixed operation labels as follows: - -| Onshape property | Operation Number | -|---|---| -| `Manufacturing Method` | `OP1` | -| `Manufacturing Method OP2` | `OP2` | -| `Manufacturing Method OP3` | `OP3` | -| `Manufacturing Method OP4` | `OP4` | - -The sync reads these values from each part's immutable released-version metadata, -so the custom properties do not need to be visible columns in the assembly BOM. -Property names and values are matched case-insensitively. `None` creates no -operation. The corresponding `Machine OP1` through `Machine OP4` fields on the -Production Requirements table and the `Machine` field on Operations rows -normalize every configured Baserow machine choice to its exact capitalization. -These additional aliases are also supported: - -- `Haas CNC` or `Haas` becomes `Haas CNC` -- `ShopSabre`, `Shop Sabre`, or `Shop Sabre CNC` becomes `Shop Sabre CNC` - -The sync creates and updates `Operation`, `Production Requirement`, `Operation -Number`, `Machine`, `Status`, and `Active in Routing`. Within each requirement's -active route, the first configured operation is `Ready`; each later operation is -`Planned` until its preceding operation is `Complete`. On subsequent syncs, a -`Planned` operation is promoted to `Ready` when its predecessor is complete, and -a prematurely ready downstream operation is returned to `Planned`. The sync -preserves `In Progress`, `Blocked`, `Needs Rework`, and `Complete`, as well as -`Machinist`, `Work Instructions`, `Started At`, `Completed At`, and `Notes`. -When an Onshape operation is changed to `None`, its existing Baserow row is -marked inactive instead of being deleted. - -### Sync Runs - -- `Started At` — primary date with time -- `Finished At` — date with time -- `Result` — single select: Running, Success, Partial, Failed -- `Source Rows` — number -- `Requirements Created` — number -- `Requirements Updated` — number -- `Requirements Unchanged` — number -- `Requirements Deactivated` — number -- `Warnings` — long text -- `Error` — long text -- `GitHub Run URL` — URL - -## Baserow token - -Create a database token with read, create, and update access to Assemblies, -Parts, Production Requirements, Operations, and Sync Runs. Add it to the GitHub repository -as an Actions secret named `BASEROW_TOKEN`. Do not commit or paste the token. -The poot-horse workflow also uses this token to upload PDF and STEP files to -Baserow before attaching them to Parts rows. - -For Baserow Cloud, no `BASEROW_API_URL` repository variable is necessary. For -self-hosting, add a repository Actions variable named `BASEROW_API_URL`, for example -`https://baserow.example.org/api`. - -Create `ONSHAPE_DOC_URL_DELTA` as a repository Actions secret containing the -Delta master assembly Main-workspace URL. `ONSHAPE_DOC_URL_EPSILON` is used only -when the poot-horse workflow opts out of its configured subassembly list and -returns to Main-workspace discovery. No manufacturing-root URL-list secret is -required because that list is stored in the poot-horse workflow. - -## Validation and cutover - -Both Baserow workflows have a manual `dry_run` input. A dry run requires the -Onshape URL and Onshape API credentials, but no Baserow credentials. It performs -release and BOM resolution, builds all records, skips every Baserow API call, -and uploads `onshape-baserow-dry-run.json` as a workflow artifact. The artifact -includes the planned Operations rows. The dry-run -job is separate from the production job and is not given the Baserow URL, token, -or table IDs. When `SYNC_CAD_FILES=true`, the dry-run artifact lists the planned -PDF and STEP filenames but does not start Onshape translations or upload files. -Manual production syncs are restricted to the default branch; -scheduled and repository-dispatch production syncs are unchanged. - -To test an implementation branch before merging, push the branch to this -repository and dispatch either existing workflow at that ref: - -```text -gh workflow run onshape_baserow_delta.yml --ref -f dry_run=true -gh workflow run onshape_baserow_poot_horse.yml --ref -f dry_run=true -``` - -The poot-horse workflow uses its configured subassembly list by default. To -test Operations for only one manufacturing root, keep list mode enabled and -provide `subassembly_urls`: - -```text -gh workflow run onshape_baserow_poot_horse.yml \ - --ref \ - -f dry_run=true \ - -f use_subassembly_list=true \ - -f sync_cad_files=false \ - -f subassembly_urls="https://frc190.onshape.com/documents/.../w/.../e/..." -``` - -Set `use_subassembly_list=false` to use Main-workspace discovery instead. In -that mode, the optional `onshape_doc_url` input replaces -`ONSHAPE_DOC_URL_EPSILON` for the manual run. - -The workflow files already exist on the default branch, which permits -`workflow_dispatch` to select the implementation branch's version. Selecting -`dry_run=false` on a non-default branch runs neither job, so it cannot start a -production Baserow sync. In list mode, the run writes the latest released BOM -for every configured root and deactivates older requirements only within those -successfully synced roots. In Main-discovery mode, a child removed from the -unreleased master is marked `Missing from Main — Review`; its requirements are -not automatically deactivated. Validate the same target selection in a dry run -first. - -After validation, dispatch the production override from the default branch: - -```text -gh workflow run onshape_baserow_poot_horse.yml \ - --ref main \ - -f dry_run=false \ - -f use_subassembly_list=true -``` - -For local use: - -```text -python pre-2027-onshape_ci/OnshapeToBaserow.py --dry-run -python pre-2027-onshape_ci/OnshapeToBaserow.py --dry-run --output-json bom-dry-run.json -``` - -1. Keep the existing Google Sheets workflow enabled. -2. Run `Sync Onshape Delta BOM to Baserow` manually with `dry_run` enabled. -3. Verify the discovered direct children, every source revision/version, - root-scoped production key, source count, aggregated quantity, configuration, - part revision, and warning. -4. Resolve any missing single-select choices or field-name mismatches. -5. Run the workflow without `dry_run`, then let both workflows run in parallel - before removing Data Fetcher and Sheets. diff --git a/pre-2027-onshape_ci/OnshapeToBaserow.py b/pre-2027-onshape_ci/OnshapeToSupabase.py similarity index 63% rename from pre-2027-onshape_ci/OnshapeToBaserow.py rename to pre-2027-onshape_ci/OnshapeToSupabase.py index 45ffe79..ba9327e 100644 --- a/pre-2027-onshape_ci/OnshapeToBaserow.py +++ b/pre-2027-onshape_ci/OnshapeToSupabase.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 -"""Synchronize an Onshape multilevel BOM into Baserow. +"""Synchronize an Onshape multilevel BOM into Supabase. Engineering-owned fields are updated when a manufacturing-root revision changes. Manufacturing status, machinist, location, QC, and disposition are intentionally -untouched. Released routing, powder-coat color, and their work queues are -sync-managed. +untouched. Released routing and powder-coat color are sync-managed. """ from __future__ import annotations @@ -17,6 +16,8 @@ import os import re import time +import uuid +from collections import Counter from dataclasses import dataclass from datetime import datetime, timezone from decimal import Decimal, InvalidOperation @@ -27,15 +28,12 @@ ASSEMBLY_NAME_RE = re.compile(r"^A-[A-Za-z0-9-]+$") -BATCH_SIZE = 100 ONSHAPE_API_VERSION = "v16" ASSEMBLY_ELEMENT_TYPE = 1 DRAWING_ELEMENT_TYPE = 2 CAD_EXPORT_CACHE_VERSION = "v2" DRAWING_PDF_FIELD = "Drawing PDF" -DRAWING_PDF_KEY_FIELD = "Drawing PDF Export Key" STEP_FILE_FIELD = "STEP File" -STEP_KEY_FIELD = "STEP Export Key" EXPORT_POLL_SECONDS = (2, 4, 8, 10) STEP_EXPORT_METHODS = frozenset( method.casefold() @@ -59,26 +57,9 @@ *OPERATION_PROPERTY_NAMES, POWDER_COAT_PROPERTY_NAME, ) -SYNC_SCHEMA_VERSION = "source-document-v1" -PRODUCTION_REQUIREMENT_MANAGED_FIELDS = ( - "Part", - "Assembly", - "Source Root", - "Source Assembly Revision", - "Required Part Revision", - "Configuration", - "Required Quantity", - "BOM Positions", - "Onshape Source", - "Source Document", - "Machine OP1", - "Machine OP2", - "Machine OP3", - "Machine OP4", - "Finishing", - "Active in BOM", -) -BASEROW_MACHINE_NAMES = ( +SYNC_SCHEMA_VERSION = "supabase-engineering-v1" +ONSHAPE_CALL_COUNTS: Counter[str] = Counter() +MACHINE_NAMES = ( "Haas CNC", "Shop Sabre CNC", "Milling Machine", @@ -100,7 +81,7 @@ ) MACHINE_NAME_ALIASES = { re.sub(r"[^a-z0-9]+", "", name.casefold()): name - for name in BASEROW_MACHINE_NAMES + for name in MACHINE_NAMES } MACHINE_NAME_ALIASES.update( { @@ -143,7 +124,6 @@ class PartExportSource: class FileExport: part_number: str field_name: str - key_field_name: str source_key: str filename: str content_type: str @@ -283,7 +263,52 @@ def onshape_headers(method: str, full_url: str) -> dict[str, str]: } +def onshape_call_category(method: str, url: str) -> str: + """Return a stable, low-cardinality label for Onshape request telemetry.""" + path = urlparse(url).path.rstrip("/") + query = parse_qs(urlparse(url).query) + if re.search(r"/assemblies/d/[^/]+/[wvm]/[^/]+/e/[^/]+/bom$", path): + return "bom" + if re.search(r"/metadata/d/[^/]+/[wvm]/[^/]+/e/[^/]+/p$", path): + return "part_metadata_bulk" + if re.search(r"/metadata/d/[^/]+/[wvm]/[^/]+/e/[^/]+/p/[^/]+$", path): + return "part_metadata_single" + if "/metadata/" in path: + return "element_metadata" + if re.search(r"/revisions/d/[^/]+$", path): + return "document_revisions" + if path.endswith("/latest") and query.get("et") == [str(ASSEMBLY_ELEMENT_TYPE)]: + return "assembly_revision" + if path.endswith("/latest") and query.get("et") == [str(DRAWING_ELEMENT_TYPE)]: + return "drawing_revision" + if re.search(r"/documents/d/[^/]+/[wvm]/[^/]+/elements$", path): + return "document_elements" + if re.search(r"/documents/[^/]+$", path): + return "document_metadata" + if "/translations" in path: + return "translation_status" if method == "GET" else "translation_create" + if "/externaldata/" in path: + return "translation_download" + return "other" + + +def record_onshape_call(method: str, url: str) -> None: + ONSHAPE_CALL_COUNTS[onshape_call_category(method, url)] += 1 + + +def reset_onshape_call_counts() -> None: + ONSHAPE_CALL_COUNTS.clear() + + +def onshape_call_summary() -> dict: + return { + "total": sum(ONSHAPE_CALL_COUNTS.values()), + "by_category": dict(sorted(ONSHAPE_CALL_COUNTS.items())), + } + + def onshape_get_json(url: str) -> dict: + record_onshape_call("GET", url) response = requests.get(url, headers=onshape_headers("GET", url), timeout=60) response.raise_for_status() payload = response.json() @@ -294,6 +319,7 @@ def onshape_get_json(url: str) -> dict: def onshape_get_optional_json(url: str) -> dict | None: """Return an Onshape JSON object, or None for a successful 204 response.""" + record_onshape_call("GET", url) response = requests.get(url, headers=onshape_headers("GET", url), timeout=60) response.raise_for_status() if response.status_code == 204: @@ -304,16 +330,8 @@ def onshape_get_optional_json(url: str) -> dict | None: return payload -def onshape_get_json_list(url: str) -> list: - response = requests.get(url, headers=onshape_headers("GET", url), timeout=60) - response.raise_for_status() - payload = response.json() - if not isinstance(payload, list): - raise RuntimeError(f"Unexpected Onshape response from {url}: expected an array") - return payload - - def onshape_post_json(url: str, body: dict) -> dict: + record_onshape_call("POST", url) response = requests.post( url, headers=onshape_headers("POST", url), json=body, timeout=60 ) @@ -325,6 +343,7 @@ def onshape_post_json(url: str, body: dict) -> dict: def onshape_download(url: str) -> bytes: + record_onshape_call("GET", url) headers = onshape_headers("GET", url) headers["Accept"] = "application/octet-stream" response = requests.get(url, headers=headers, timeout=120) @@ -399,20 +418,6 @@ def fetch_latest_discovered_assembly_revision( ) -def fetch_latest_drawing_revision( - reference: OnshapeDocumentReference, part_number: str -) -> dict | None: - """Fetch the latest released drawing revision for a company-owned part number.""" - encoded_part_number = quote(part_number, safe="") - endpoint = ( - f"{reference.base_url}/api/{ONSHAPE_API_VERSION}/revisions/d/{reference.did}/" - f"p/{encoded_part_number}/latest" - ) - return onshape_get_optional_json( - f"{endpoint}?{urlencode({'et': DRAWING_ELEMENT_TYPE})}" - ) - - def released_drawing_url( reference: OnshapeDocumentReference, part_number: str, latest: dict ) -> str: @@ -809,25 +814,11 @@ def source_document_reference( return OnshapeDocumentReference(base_url, did, wvm_type, wvm_id) -def fetch_document_elements(reference: OnshapeDocumentReference) -> list[dict]: - endpoint = ( - f"{reference.base_url}/api/{ONSHAPE_API_VERSION}/documents/d/" - f"{reference.did}/{reference.wvm_type}/{reference.wvm_id}/elements" - ) - elements = onshape_get_json_list(endpoint) - if not all(isinstance(element, dict) for element in elements): - raise RuntimeError( - f"Unexpected Onshape elements response for document {reference.did}" - ) - return elements - - -def fetch_element_metadata( - reference: OnshapeDocumentReference, element_id: str -) -> dict: +def fetch_document_revisions(base_url: str, document_id: str) -> dict: + """Fetch every released revision in one Onshape document.""" endpoint = ( - f"{reference.base_url}/api/{ONSHAPE_API_VERSION}/metadata/d/" - f"{reference.did}/{reference.wvm_type}/{reference.wvm_id}/e/{element_id}" + f"{base_url.rstrip('/')}/api/{ONSHAPE_API_VERSION}/revisions/d/" + f"{quote(document_id, safe='')}" ) return onshape_get_json(endpoint) @@ -884,7 +875,7 @@ def discover_released_manufacturing_roots( if not latest or not str(latest.get("versionId") or "").strip(): warnings.append( f"Direct child {candidate_number} has no released assembly revision; " - "existing Baserow requirements were left unchanged" + "existing Supabase requirements were left unchanged" ) continue released = released_assembly_from_revision(latest) @@ -906,34 +897,15 @@ def discover_released_manufacturing_roots( return roots, sorted(set(warnings)) -def is_drawing_element(element: dict) -> bool: - element_type = str( - element.get("elementType") or element.get("type") or "" - ).strip().casefold() - if element_type == "drawing": - return True - structured_markers = ( - element.get("mimeType"), - element.get("dataType"), - element.get("applicationType"), - ) - if any( - "drawing" in str(marker or "").casefold() for marker in structured_markers - ): - return True - return ( - element_type == "application" or element_type.isdigit() - ) and "drawing" in str(element.get("name") or "").casefold() - - def drawing_urls_for_parts( items: list[dict], prefixes: list[str], default_base_url: str, extra_references: list[OnshapeDocumentReference] | None = None, + revision_cache: dict[tuple[str, str], dict] | None = None, ) -> tuple[dict[str, str], list[str]]: - """Find drawing tabs, then resolve each PDF source to its own latest release.""" - expected_by_reference: dict[OnshapeDocumentReference, dict[str, str]] = {} + """Resolve released drawings with one revision request per source document.""" + expected_by_document: dict[tuple[str, str], dict[str, str]] = {} all_expected: dict[str, str] = {} for row in items: part_number = str(row.get("partNumber") or "").strip() @@ -945,54 +917,89 @@ def drawing_urls_for_parts( reference = source_document_reference(row.get("itemSource"), default_base_url) if reference is None: continue - expected_by_reference.setdefault(reference, {})[ + document_key = (reference.base_url.rstrip("/"), reference.did) + expected_by_document.setdefault(document_key, {})[ normalized_part_number(part_number) ] = part_number + if not all_expected: + return {}, [] + for reference in set(extra_references or []): - expected_by_reference.setdefault(reference, {}).update(all_expected) + document_key = (reference.base_url.rstrip("/"), reference.did) + expected_by_document.setdefault(document_key, {}).update(all_expected) + + cache = revision_cache if revision_cache is not None else {} + drawing_candidates: dict[str, set[str]] = {} + for document_key, expected in expected_by_document.items(): + if document_key not in cache: + cache[document_key] = fetch_document_revisions(*document_key) + payload = cache[document_key] + revision_items = next( + ( + payload.get(key) + for key in ("items", "revisions", "results") + if isinstance(payload.get(key), list) + ), + None, + ) + if revision_items is None: + raise RuntimeError( + f"Unexpected Onshape revisions response for document " + f"{document_key[1]}: no items array" + ) - drawing_candidates: dict[str, set[OnshapeDocumentReference]] = {} - for reference, expected in expected_by_reference.items(): - for element in fetch_document_elements(reference): - if not is_drawing_element(element): + latest_by_part: dict[str, dict] = {} + for revision_item in revision_items: + if not isinstance(revision_item, dict): continue - element_id = str(element.get("id") or element.get("elementId") or "").strip() - if not element_id: + element_type = str(revision_item.get("elementType") or "").casefold() + if element_type not in (str(DRAWING_ELEMENT_TYPE), "drawing"): continue - candidate_keys = { - normalized_part_number(element.get("name")), - normalized_part_number(element.get("partNumber")), - } - if not any(candidate_key in expected for candidate_key in candidate_keys): - metadata = fetch_element_metadata(reference, element_id) - candidate_keys.add( - normalized_part_number(metadata_property(metadata, "Part number")) + candidate_key = normalized_part_number(revision_item.get("partNumber")) + if candidate_key not in expected: + continue + current = latest_by_part.get(candidate_key) + sort_key = ( + str( + revision_item.get("releaseCreatedDate") + or revision_item.get("createdAt") + or "" + ), + str(revision_item.get("revision") or ""), + str(revision_item.get("versionId") or ""), + str(revision_item.get("elementId") or ""), + ) + current_key = ( + ( + str( + current.get("releaseCreatedDate") + or current.get("createdAt") + or "" + ), + str(current.get("revision") or ""), + str(current.get("versionId") or ""), + str(current.get("elementId") or ""), ) - for candidate_key in candidate_keys - {""}: - part_number = expected.get(candidate_key) - if not part_number: - continue - drawing_candidates.setdefault(part_number, set()).add(reference) + if current is not None + else None + ) + if current_key is None or sort_key > current_key: + latest_by_part[candidate_key] = revision_item + + reference = OnshapeDocumentReference( + document_key[0], document_key[1], "v", "unused" + ) + for candidate_key, latest in latest_by_part.items(): + part_number = expected[candidate_key] + drawing_candidates.setdefault(part_number, set()).add( + released_drawing_url(reference, part_number, latest) + ) drawing_urls: dict[str, str] = {} warnings: list[str] = [] - release_cache: dict[tuple[str, str, str], dict | None] = {} - for part_number in sorted(drawing_candidates): - urls: set[str] = set() - for reference in drawing_candidates[part_number]: - cache_key = ( - reference.base_url, - reference.did, - normalized_part_number(part_number), - ) - if cache_key not in release_cache: - release_cache[cache_key] = fetch_latest_drawing_revision( - reference, part_number - ) - latest = release_cache[cache_key] - if latest is not None: - urls.add(released_drawing_url(reference, part_number, latest)) + for part_number in sorted(all_expected.values()): + urls = drawing_candidates.get(part_number, set()) if len(urls) == 1: drawing_urls[part_number] = next(iter(urls)) elif not urls: @@ -1115,7 +1122,6 @@ def build_file_exports( FileExport( part_number=part_number, field_name=DRAWING_PDF_FIELD, - key_field_name=DRAWING_PDF_KEY_FIELD, source_key=export_source_key("drawing", coordinates), filename=f"{stem}.pdf", content_type="application/pdf", @@ -1167,7 +1173,6 @@ def build_file_exports( FileExport( part_number=part_number, field_name=STEP_FILE_FIELD, - key_field_name=STEP_KEY_FIELD, source_key=source_key, filename=f"{stem}.step", content_type="application/step", @@ -1217,7 +1222,7 @@ def row_property(row: dict, property_name: str): def operation_machine_name(value) -> str: - """Return the exact Baserow machine choice for an Onshape method value.""" + """Return the exact Supabase machine choice for an Onshape method value.""" machine = str(value or "").strip() normalized = normalized_property_name(machine) if normalized in ("", "none", "selectvalue"): @@ -1226,7 +1231,7 @@ def operation_machine_name(value) -> str: def powder_coat_color(value) -> str: - """Return an exact Baserow choice for the released Onshape color.""" + """Return an exact Supabase choice for the released Onshape color.""" normalized = normalized_property_name(value) if normalized in ("", "none", "selectvalue"): return "None" @@ -1258,6 +1263,35 @@ def fetch_part_metadata(item_source: dict, default_base_url: str) -> dict | None return onshape_get_json(f"{endpoint}?{urlencode(params)}") +def fetch_parts_metadata( + reference: OnshapeDocumentReference, + element_id: str, + configuration: str, +) -> dict: + """Read all part metadata for one immutable Part Studio configuration.""" + endpoint = ( + f"{reference.base_url}/api/{ONSHAPE_API_VERSION}/metadata/d/{reference.did}/" + f"{reference.wvm_type}/{reference.wvm_id}/e/{quote(element_id, safe='')}/p" + ) + params = { + "includeComputedProperties": "true", + "includeComputedAssemblyProperties": "false", + "thumbnail": "false", + } + if configuration != "default": + params["configuration"] = configuration + return onshape_get_json(f"{endpoint}?{urlencode(params)}") + + +def parts_metadata_items(payload: dict) -> list[dict]: + """Return the typed metadata entries from the bulk metadata response.""" + for key in ("items", "parts", "objects", "results"): + items = payload.get(key) + if isinstance(items, list) and all(isinstance(item, dict) for item in items): + return items + raise RuntimeError("Unexpected Onshape bulk part metadata response: no items array") + + def operation_metadata_values(payload: dict) -> dict[str, object]: values = {} properties = payload.get("properties") @@ -1273,13 +1307,19 @@ def operation_metadata_values(payload: dict) -> dict[str, object]: def hydrate_operation_properties( - items: list[dict], prefixes: list[str], default_base_url: str + items: list[dict], + prefixes: list[str], + default_base_url: str, + bulk_cache: dict[tuple, dict[str, dict]] | None = None, + single_cache: dict[tuple, dict | None] | None = None, ) -> list[dict]: - """Overlay released part metadata so operation properties need not be BOM columns.""" - cache: dict[tuple, dict | None] = {} - hydrated = [] - for original in items: - row = dict(original) + """Overlay routing metadata using one request per Part Studio/configuration.""" + bulk_metadata_cache = bulk_cache if bulk_cache is not None else {} + fallback_cache = single_cache if single_cache is not None else {} + hydrated = [dict(original) for original in items] + rows_by_group: dict[tuple, list[tuple[dict, dict, str]]] = {} + + for row in hydrated: part_number = str(row.get("partNumber") or "").strip() item_source = row.get("itemSource") if part_number and ( @@ -1290,27 +1330,49 @@ def hydrate_operation_properties( part_id = str(item_source.get("partId") or "").strip() _, configuration = source_url_and_configuration(item_source) if reference is not None and element_id and part_id: - cache_key = ( + group_key = ( reference.base_url, reference.did, reference.wvm_type, reference.wvm_id, element_id, - part_id, configuration, ) - if cache_key not in cache: - cache[cache_key] = fetch_part_metadata( + rows_by_group.setdefault(group_key, []).append( + (row, item_source, part_id) + ) + + for group_key, group_rows in rows_by_group.items(): + if group_key not in bulk_metadata_cache: + reference = OnshapeDocumentReference( + group_key[0], group_key[1], group_key[2], group_key[3] + ) + payload = fetch_parts_metadata( + reference, group_key[4], group_key[5] + ) + bulk_metadata_cache[group_key] = { + str(item.get("partId") or item.get("id") or ""): item + for item in parts_metadata_items(payload) + if str(item.get("partId") or item.get("id") or "").strip() + } + metadata_by_part = bulk_metadata_cache[group_key] + + for row, item_source, part_id in group_rows: + metadata = metadata_by_part.get(part_id) + if metadata is None: + fallback_key = (*group_key, part_id) + if fallback_key not in fallback_cache: + fallback_cache[fallback_key] = fetch_part_metadata( item_source, default_base_url ) - metadata = cache[cache_key] - if metadata is not None: - metadata_values = operation_metadata_values(metadata) - for property_name in HYDRATED_PART_PROPERTY_NAMES: - property_key = normalized_property_name(property_name) - if property_key in metadata_values: - row[property_name] = metadata_values[property_key] - hydrated.append(row) + metadata = fallback_cache[fallback_key] + if metadata is None: + continue + metadata_values = operation_metadata_values(metadata) + for property_name in HYDRATED_PART_PROPERTY_NAMES: + property_key = normalized_property_name(property_name) + if property_key in metadata_values: + row[property_name] = metadata_values[property_key] return hydrated @@ -1324,7 +1386,7 @@ def operation_machines_from_row(row: dict) -> tuple[tuple[str, str], ...]: def production_requirement_machine_fields(row: dict) -> dict: - """Return released routing values using exact Baserow choice names.""" + """Return released routing values using exact Supabase choice names.""" fields = {} for index, property_name in enumerate(OPERATION_PROPERTY_NAMES, start=1): machine = operation_machine_name(row_property(row, property_name)) @@ -1352,54 +1414,6 @@ def build_operation_records(requirements: list[dict]) -> list[dict]: return operations -def select_option_value(value) -> str: - """Return the displayed value from a Baserow single-select response.""" - if isinstance(value, dict): - value = value.get("value") - return str(value or "").strip() - - -def operation_sequence(operation: dict) -> int: - """Return the numeric position of an OP1 through OP4 operation.""" - label = select_option_value(operation.get("Operation Number")).upper() - if label.startswith("OP") and label[2:].isdigit(): - return int(label[2:]) - return 999 - - -def operation_statuses_for_routes( - operations: list[dict], existing_rows: list[dict] -) -> dict[str, str]: - """Gate each operation on completion of the preceding active route step. - - Planned and Ready are sync-managed queue states. Manufacturing-owned states - are preserved, including In Progress, Blocked, Needs Rework, and Complete. - """ - existing_by_key = { - str(row.get("Operation") or ""): row for row in existing_rows - } - routes: dict[str, list[dict]] = {} - for operation in operations: - production_key = str(operation.get("production_key") or "").strip() - routes.setdefault(production_key, []).append(operation) - - statuses = {} - for route in routes.values(): - predecessor_complete = True - for operation in sorted(route, key=operation_sequence): - operation_key = str(operation.get("Operation") or "") - current_status = select_option_value( - existing_by_key.get(operation_key, {}).get("Status") - ) - if current_status in ("", "Planned", "Ready"): - status = "Ready" if predecessor_complete else "Planned" - else: - status = current_status - statuses[operation_key] = status - predecessor_complete = status == "Complete" - return statuses - - def build_records( items: list[dict], prefixes: list[str], @@ -1434,7 +1448,7 @@ def build_records( "Manufacturing Method": str(row.get("manufacturingmethod") or "").strip(), "Vendor": str(row.get("vendor") or "").strip(), "Revision": str(row.get("revision") or "").strip(), - "OnShape Text": str(row.get("state") or "").strip(), + "OnShape Text": source_url, "Category": str(row.get("category") or "").strip(), "Active": True, } @@ -1513,134 +1527,201 @@ def build_records( return list(parts.values()), list(requirements.values()), sorted(set(warnings)) -class BaserowClient: - def __init__(self, base_url: str, token: str): +class SupabaseClient: + """Only public engineering RPCs and private Storage; no table REST access.""" + + def __init__(self, base_url: str, secret: str): + if not base_url.startswith("https://"): + raise ValueError("NEXT_PUBLIC_SUPABASE_URL must use HTTPS") self.base_url = base_url.rstrip("/") self.session = requests.Session() - self.session.headers.update({"Authorization": f"Token {token}"}) - - def _url(self, table_id: int, suffix: str = "") -> str: - return f"{self.base_url}/database/rows/table/{table_id}/{suffix}?user_field_names=true" - - def list_rows(self, table_id: int) -> list[dict]: - rows = [] - page = 1 - while True: - response = self.session.get(self._url(table_id), params={"user_field_names": "true", "page": page, "size": 200}, timeout=60) - response.raise_for_status() - payload = response.json() - rows.extend(payload.get("results", [])) - if not payload.get("next"): - return rows - page += 1 - - def create_one(self, table_id: int, fields: dict) -> dict: - response = self.session.post(self._url(table_id), json=fields, timeout=60) - response.raise_for_status() - return response.json() + self.session.max_redirects = 0 + self.session.headers.update({"apikey": secret}) + # Supabase secret keys are not JWTs. Legacy service-role JWTs are supported. + if not secret.startswith("sb_secret_"): + self.session.headers["Authorization"] = f"Bearer {secret}" + + @classmethod + def from_env(cls): + return cls(require_env("NEXT_PUBLIC_SUPABASE_URL"), require_env("SUPABASE_SECRET_KEY")) - def update_one(self, table_id: int, row_id: int, fields: dict) -> dict: - response = self.session.patch(self._url(table_id, str(row_id) + "/"), json=fields, timeout=60) + def rpc(self, name: str, **params): + response = self.session.post( + f"{self.base_url}/rest/v1/rpc/{name}", json=params, timeout=120 + ) response.raise_for_status() return response.json() - @staticmethod - def _raise_batch_error_with_context( - response, table_id: int, operation: str, items: list[dict] - ) -> None: - try: - response.raise_for_status() - except Exception as exc: - try: - response_detail = response.json() - except Exception: - response_detail = str(getattr(response, "text", "") or "").strip() - identifying_fields = ( - "id", - "Production Key", - "Part Number", - "Assembly Number", - "Operation", - "Operation Number", - "Machine", - "Machine OP1", - "Machine OP2", - "Machine OP3", - "Machine OP4", - ) - item_identifiers = [] - for batch_index, item in enumerate(items): - identifier = {"batch_index": batch_index} - for field in identifying_fields: - value = item.get(field) - if value not in (None, "", [], {}): - identifier[field] = value - item_identifiers.append(identifier) - detail_text = json.dumps( - response_detail, sort_keys=True, default=str - )[:10000] - identifiers_text = json.dumps( - item_identifiers, sort_keys=True, default=str - )[:20000] - raise RuntimeError( - f"Baserow batch {operation} failed for table {table_id} " - f"with HTTP {getattr(response, 'status_code', 'unknown')}; " - f"response={detail_text}; " - f"batch_item_identifiers={identifiers_text}" - ) from exc + def begin_run(self) -> str: + run_id = str(uuid.uuid4()) + self.rpc("manufacturing_begin_engineering_sync", p_run_id=run_id, + p_run_url=os.environ.get("GITHUB_RUN_URL", "")) + return run_id - def batch_create(self, table_id: int, items: list[dict]) -> list[dict]: - created = [] - for start in range(0, len(items), BATCH_SIZE): - batch = items[start:start+BATCH_SIZE] - response = self.session.post( - self._url(table_id, "batch/"), - json={"items": batch}, - timeout=60, - ) - self._raise_batch_error_with_context( - response, table_id, "create", batch - ) - created.extend(response.json().get("items", [])) - return created - - def batch_update(self, table_id: int, items: list[dict]) -> list[dict]: - updated = [] - for start in range(0, len(items), BATCH_SIZE): - batch = items[start:start+BATCH_SIZE] - response = self.session.patch( - self._url(table_id, "batch/"), - json={"items": batch}, - timeout=60, - ) - self._raise_batch_error_with_context( - response, table_id, "update", batch - ) - updated.extend(response.json().get("items", [])) - return updated + def finish_run(self, run_id: str, status: str, summary: dict): + return self.rpc("manufacturing_finish_engineering_sync", p_run_id=run_id, + p_status=status, p_summary=summary) + + def root_state(self) -> list[dict]: + return self.rpc("manufacturing_engineering_sync_state") + + def attachment_state(self, part_numbers: list[str]) -> list[dict]: + return self.rpc("manufacturing_engineering_file_state", p_part_numbers=part_numbers) def upload_file(self, filename: str, content: bytes, content_type: str) -> dict: - response = self.session.post( - f"{self.base_url}/user-files/upload-file/", - files={"file": (filename, content, content_type)}, - timeout=120, - ) - response.raise_for_status() - payload = response.json() - if not isinstance(payload, dict) or not payload.get("name"): - raise RuntimeError("Unexpected Baserow file upload response: no file name") - return payload - - -def all_root_revisions_are_current( - released_roots: list[ReleasedAssembly], discovery_master: str = "" -) -> bool: - """Return whether Baserow already represents every resolved root revision.""" - client = BaserowClient( - require_env("BASEROW_API_URL"), require_env("BASEROW_TOKEN") - ) - assemblies_table_id = int(require_env("BASEROW_ASSEMBLIES_TABLE_ID")) - rows = client.list_rows(assemblies_table_id) + digest = hashlib.sha256(content).hexdigest() + extension = "pdf" if content_type == "application/pdf" else "step" + path = f"sha256/{digest[:2]}/{digest}.{extension}" + url = f"{self.base_url}/storage/v1/object/manufacturing-files/{path}" + response = self.session.post(url, data=content, headers={ + "Content-Type": content_type, "x-upsert": "false" + }, timeout=120) + if response.status_code not in (200, 201): + # Only a genuine duplicate is acceptable. Verify its bytes as well. + try: + duplicate = str(response.json().get("error", "")) in ("Duplicate", "ResourceAlreadyExists") + except (ValueError, AttributeError): + duplicate = False + if not duplicate: + response.raise_for_status() + raise RuntimeError("Storage upload failed") + verified = self.session.get(url, timeout=120) + verified.raise_for_status() + if hashlib.sha256(verified.content).hexdigest() != digest: + raise RuntimeError("Storage verification failed") + return {"original_name": filename, "content_type": content_type, + "byte_size": len(content), "sha256": digest, + "storage_bucket": "manufacturing-files", "storage_path": path, + "verified_at": utc_now()} + + +# Display names are retained in dry-run records. This single boundary maps them +# to the engineering columns in the authoritative manufacturing model. +ENGINEERING_COLUMNS = { + "assemblies": { + "Assembly Number": "assembly_number", "Subsystem Name": "subsystem_name", + "Active": "active", "Sync Schema Version": "sync_schema_version", + "Latest Released Revision": "latest_released_revision", + "Master Baseline Revision": "master_baseline_revision", + "Integration Status": "integration_status", "Discovery Master": "discovery_master", + "Onshape Source": "onshape_url", "Last Synced At": "last_synced_at", + }, + "parts": { + "Part Number": "part_number", "Name": "name", "Description": "description", + "Material": "material", "Manufacturing Method": "manufacturing_method", + "Vendor": "vendor", "Revision": "revision", "OnShape Text": "onshape_url", + "Category": "category", "Onshape Drawing": "drawing_url", "Active": "active", + }, + "requirements": { + "Production Key": "production_key", "part_number": "part_number", + "assembly_number": "assembly_number", "Configuration": "configuration", + "Required Quantity": "required_quantity", "BOM Positions": "bom_positions", + "Onshape Source": "onshape_url", "Source Document": "source_document", + "Source Root": "source_root", "Source Assembly Revision": "source_assembly_revision", + "Required Part Revision": "required_part_revision", "Machine OP1": "machine_op1", + "Machine OP2": "machine_op2", "Machine OP3": "machine_op3", "Machine OP4": "machine_op4", + "Finishing": "finishing", "Active in BOM": "active_in_bom", + }, + "operations": { + "Operation": "operation_key", "production_key": "production_key", + "Operation Number": "operation_number", "Machine": "machine", + "Active in Routing": "active_in_routing", + }, +} + + +def engineering_rows(entity: str, rows: list[dict]) -> list[dict]: + return [{column: row[field] for field, column in ENGINEERING_COLUMNS[entity].items() + if field in row} for row in rows] + + +def attach_exported_files(client, parts, existing_rows, exports_by_part, warnings): + """Stage complete file groups. Failed groups leave the catalog and keys intact.""" + existing = {(row["part_number"], row["kind"]): row for row in existing_rows} + groups, cached = [], 0 + for part in parts: + number = part["Part Number"] + for field, kind in ((DRAWING_PDF_FIELD, "drawing-pdf"), (STEP_FILE_FIELD, "step")): + exports = sorted((e for e in exports_by_part.get(number, []) if e.field_name == field), + key=lambda e: e.source_key) + if not exports: + continue + key = aggregate_export_key(exports) + state = existing.get((number, kind), {}) + if state.get("export_key") == key and state.get("file_count") == len(exports): + cached += 1 + continue + try: + files = [] + for export in exports: + initial = start_file_translation(export) + completed = wait_for_translation(export, initial) + content = download_translation(export, completed) + metadata = {**initial, **{k: v for k, v in completed.items() if v not in (None, "")}} + uploaded = client.upload_file(completed_export_filename(export, metadata), content, export.content_type) + # Stable per-part source identity fits the catalog's unique source_url. + files.append({**uploaded, + "source_url": export.endpoint + "?" + urlencode({"part_number": number, "export_key": export.source_key}), + "source_metadata": {"export_key": export.source_key, "request": export.request_body}, + }) + groups.append({"part_number": number, "kind": kind, "export_key": key, "files": files}) + except Exception as exc: + warnings.append(f"Could not refresh {field} for {number}: {exc}") + return groups, cached + + +def sync_to_supabase(parts, requirements, warnings, source_rows, exports_by_part, + sync_cad_files, operations=None, assembly_records=None, + synced_roots=None, discovery_master="", discovered_roots=None, + discovery_complete=True, run_id=None): + client = SupabaseClient.from_env() + run_id = run_id or client.begin_run() + try: + assemblies = {r["assembly_number"]: {"Assembly Number": r["assembly_number"], "Active": True} + for r in requirements if r.get("assembly_number")} + assemblies.update({a["Assembly Number"]: a for a in (assembly_records or [])}) + groups, cached = [], 0 + initial_warnings = len(warnings) + if sync_cad_files: + state = client.attachment_state([p["Part Number"] for p in parts]) + groups, cached = attach_exported_files(client, parts, state, exports_by_part, warnings) + payload = { + "assemblies": engineering_rows("assemblies", list(assemblies.values())), + "parts": engineering_rows("parts", parts), + "requirements": engineering_rows("requirements", requirements), + "operations": [{**r, "work_type": "Manufacturing"} + for r in engineering_rows("operations", operations or [])], + "finishing": [{"production_key": r["Production Key"], "color": r["Finishing"], + "required_quantity": r["Required Quantity"], "active": True} + for r in requirements if r.get("Finishing") in ("Red", "Black")], + "attachments": groups, + "synced_roots": sorted(synced_roots or []), + "discovered_roots": sorted(discovered_roots or synced_roots or []), + "discovery_master": discovery_master, "discovery_complete": discovery_complete, + # Retry incomplete exports on the next run without falsely marking CAD current. + "cad_synced": sync_cad_files and len(warnings) == initial_warnings, + "warnings": sorted(set(warnings)), "source_rows": source_rows, + "file_groups_cached": cached, + } + result = client.rpc("manufacturing_apply_engineering_sync", p_run_id=run_id, p_payload=payload) + if not isinstance(result, dict) or result.get("status") not in ("success", "partial", "failed"): + raise RuntimeError("Unexpected engineering sync RPC result") + if result.get("status") == "failed": + raise RuntimeError("Engineering transaction rolled back: " + result.get("error", "unknown error")) + return result + except Exception as exc: + try: + client.finish_run(run_id, "failed", {"error": str(exc)[:10000]}) + except Exception as audit_error: + print(f"WARNING: Could not record sync failure: {audit_error}") + raise + + +def stale_root_revisions( + released_roots: list[ReleasedAssembly], discovery_master: str = "", *, sync_cad_files: bool = False +) -> tuple[set[str], bool]: + """Return stale normalized root numbers and discovery-membership status.""" + rows = SupabaseClient.from_env().root_state() rows_by_number = { normalized_part_number(row.get("Assembly Number")): row for row in rows @@ -1650,9 +1731,11 @@ def all_root_revisions_are_current( root_numbers = { normalized_part_number(released.part_number) for released in released_roots } + stale: set[str] = set() for released in released_roots: revision = str(released.revision or "").strip() - current = rows_by_number.get(normalized_part_number(released.part_number)) + root_number = normalized_part_number(released.part_number) + current = rows_by_number.get(root_number) if ( not revision or current is None @@ -1660,9 +1743,11 @@ def all_root_revisions_are_current( != revision or str(current.get("Sync Schema Version") or "").strip() != SYNC_SCHEMA_VERSION + or (sync_cad_files and not current.get("CAD Synced")) ): - return False + stale.add(root_number) + membership_changed = False if discovery_master: previously_present = { normalized_part_number(row.get("Assembly Number")) @@ -1672,9 +1757,9 @@ def all_root_revisions_are_current( != "Missing from Main — Review" } if previously_present != root_numbers: - return False + membership_changed = True - return True + return stale, membership_changed def aggregate_export_key(exports: list[FileExport]) -> str: @@ -1682,17 +1767,6 @@ def aggregate_export_key(exports: list[FileExport]) -> str: return hashlib.sha256(encoded).hexdigest() -def existing_file_is_current( - row: dict, field_name: str, key_field_name: str, exports: list[FileExport] -) -> bool: - files = row.get(field_name) - return ( - str(row.get(key_field_name) or "") == aggregate_export_key(exports) - and isinstance(files, list) - and len(files) == len(exports) - ) - - def start_file_translation(export: FileExport) -> dict: payload = onshape_post_json(export.endpoint, export.request_body) translation_id = str(payload.get("id") or "").strip() @@ -1748,16 +1822,6 @@ def download_translation(export: FileExport, translation: dict) -> bytes: return content -def baserow_file_references(value) -> list[dict[str, str]]: - if not isinstance(value, list): - return [] - return [ - {"name": str(item["name"])} - for item in value - if isinstance(item, dict) and item.get("name") - ] - - def completed_export_filename(export: FileExport, translation: dict) -> str: filename = str( translation.get("exportRuleFileName") @@ -1775,553 +1839,6 @@ def completed_export_filename(export: FileExport, translation: dict) -> str: return filename -def attach_exported_files( - client: BaserowClient, - parts: list[dict], - existing_rows: list[dict], - exports_by_part: dict[str, list[FileExport]], - warnings: list[str], -) -> tuple[int, int]: - """Export changed CAD files and add Baserow file values to desired part rows.""" - existing_by_part = { - str(row.get("Part Number") or ""): row for row in existing_rows - } - part_by_number = {part["Part Number"]: part for part in parts} - pending_groups: list[tuple[dict, list[FileExport]]] = [] - cached_groups = 0 - - for part_number, part in part_by_number.items(): - by_field: dict[str, list[FileExport]] = {} - for export in exports_by_part.get(part_number, []): - by_field.setdefault(export.field_name, []).append(export) - existing = existing_by_part.get(part_number, {}) - for field_name, key_field_name in ( - (DRAWING_PDF_FIELD, DRAWING_PDF_KEY_FIELD), - (STEP_FILE_FIELD, STEP_KEY_FIELD), - ): - group = by_field.get(field_name, []) - part[field_name] = baserow_file_references(existing.get(field_name)) - part[key_field_name] = str(existing.get(key_field_name) or "") - if not group: - continue - elif existing_file_is_current(existing, field_name, key_field_name, group): - cached_groups += 1 - else: - pending_groups.append((part, group)) - - started: list[tuple[dict, list[FileExport], list[tuple[FileExport, dict]]]] = [] - for part, group in pending_groups: - translations: list[tuple[FileExport, dict]] = [] - try: - for export in group: - translations.append((export, start_file_translation(export))) - started.append((part, group, translations)) - except Exception as exc: - warnings.append( - f"Could not start {group[0].field_name} export for " - f"{group[0].part_number}: {exc}" - ) - - uploaded_groups = 0 - for part, group, translations in started: - try: - attachments = [] - for export, initial in translations: - completed = wait_for_translation(export, initial) - content = download_translation(export, completed) - filename_metadata = { - **initial, - **{ - key: value - for key, value in completed.items() - if value not in (None, "") - }, - } - uploaded = client.upload_file( - completed_export_filename(export, filename_metadata), - content, - export.content_type, - ) - attachments.append({"name": uploaded["name"]}) - part[group[0].field_name] = attachments - part[group[0].key_field_name] = aggregate_export_key(group) - uploaded_groups += 1 - except Exception as exc: - warnings.append( - f"Could not refresh {group[0].field_name} for " - f"{group[0].part_number}: {exc}" - ) - return uploaded_groups, cached_groups - - -def comparable(value): - if isinstance(value, list): - normalized = [] - for item in value: - if isinstance(item, dict): - item = item.get("id", item.get("name", item)) - normalized.append(json.dumps(item, sort_keys=True, default=str)) - return sorted(normalized) - if isinstance(value, dict) and "value" in value: - return comparable(value.get("value")) - return value if value is not None else "" - - -def changed(existing: dict, desired: dict, fields: tuple[str, ...]) -> bool: - return any(comparable(existing.get(field)) != comparable(desired.get(field)) for field in fields) - - -def linked_row_ids(value) -> set[int]: - ids = set() - for item in value if isinstance(value, list) else []: - row_id = item.get("id") if isinstance(item, dict) else item - try: - ids.add(int(row_id)) - except (TypeError, ValueError): - continue - return ids - - -def upsert_table( - client: BaserowClient, - table_id: int, - key_field: str, - desired: list[dict], - update_fields: tuple[str, ...], - change_flag_field: str | None = None, -): - existing = client.list_rows(table_id) - by_key = {str(row.get(key_field) or ""): row for row in existing} - creates, updates = [], [] - for fields in desired: - current = by_key.get(str(fields[key_field])) - if current is None: - creates.append({**fields, **({change_flag_field: False} if change_flag_field else {})}) - elif changed(current, fields, update_fields): - updates.append({"id": current["id"], **fields, **({change_flag_field: True} if change_flag_field else {})}) - created = client.batch_create(table_id, creates) if creates else [] - updated = client.batch_update(table_id, updates) if updates else [] - return len(created), len(updated), len(desired) - len(creates) - len(updates) - - -def sync_to_baserow( - parts: list[dict], - requirements: list[dict], - warnings: list[str], - source_rows: int, - exports_by_part: dict[str, list[FileExport]], - sync_cad_files: bool, - operations: list[dict] | None = None, - assembly_records: list[dict] | None = None, - synced_roots: set[str] | None = None, - discovery_master: str = "", -) -> dict: - client = BaserowClient(require_env("BASEROW_API_URL"), require_env("BASEROW_TOKEN")) - table_ids = { - "sync": int(require_env("BASEROW_SYNC_RUNS_TABLE_ID")), - "parts": int(require_env("BASEROW_PARTS_TABLE_ID")), - "requirements": int(require_env("BASEROW_REQUIREMENTS_TABLE_ID")), - "operations": int(require_env("BASEROW_OPERATIONS_TABLE_ID")), - "assemblies": int(require_env("BASEROW_ASSEMBLIES_TABLE_ID")), - "finishing": int(require_env("BASEROW_FINISHING_TABLE_ID")), - } - started = utc_now() - run = client.create_one(table_ids["sync"], {"Started At": started, "Result": "Running", "Source Rows": source_rows}) - try: - now = utc_now() - assembly_records = assembly_records or [] - operations = operations or [] - synced_roots = synced_roots or { - str(requirement.get("Source Root") or "") - for requirement in requirements - if requirement.get("Source Root") - } - assembly_numbers = { - str(requirement.get("assembly_number") or "") - for requirement in requirements - if requirement.get("assembly_number") - } | { - str(assembly.get("Assembly Number") or "") - for assembly in assembly_records - if assembly.get("Assembly Number") - } - assemblies = [ - {"Assembly Number": number, "Active": True} - for number in sorted(assembly_numbers) - ] - assembly_fields = ("Assembly Number", "Active") - upsert_table(client, table_ids["assemblies"], "Assembly Number", assemblies, assembly_fields) - assembly_rows = client.list_rows(table_ids["assemblies"]) - root_assembly_fields: tuple[str, ...] = () - supported_assembly_records: list[dict] = [] - if assembly_records: - available_assembly_fields = { - field for row in assembly_rows for field in row - } - candidate_assembly_fields = ( - "Subsystem Name", - "Active", - "Latest Released Revision", - "Master Baseline Revision", - "Integration Status", - "Discovery Master", - "Onshape Source", - "Last Synced At", - "Sync Schema Version", - ) - root_assembly_fields = tuple( - field - for field in candidate_assembly_fields - if field in available_assembly_fields - ) - supported_assembly_records = [ - { - field: value - for field, value in assembly.items() - if field == "Assembly Number" or field in root_assembly_fields - } - for assembly in assembly_records - ] - - missing_from_master = [] - if discovery_master and assembly_rows: - available_assembly_fields = { - field for row in assembly_rows for field in row - } - if { - "Discovery Master", - "Integration Status", - }.issubset(available_assembly_fields): - missing_from_master = [ - { - "id": row["id"], - "Integration Status": "Missing from Main — Review", - } - for row in assembly_rows - if str(row.get("Discovery Master") or "").strip() - == discovery_master - and str(row.get("Assembly Number") or "").strip() - not in synced_roots - and str(row.get("Integration Status") or "").strip() - != "Missing from Main — Review" - ] - if missing_from_master: - client.batch_update( - table_ids["assemblies"], missing_from_master - ) - assembly_ids = {str(r.get("Assembly Number") or ""): r["id"] for r in assembly_rows} - - part_rows = client.list_rows(table_ids["parts"]) - files_uploaded = files_cached = 0 - if sync_cad_files: - required_file_fields = ( - DRAWING_PDF_FIELD, - DRAWING_PDF_KEY_FIELD, - STEP_FILE_FIELD, - STEP_KEY_FIELD, - ) - missing_fields = [ - field - for field in required_file_fields - if part_rows and not any(field in row for row in part_rows) - ] - if missing_fields: - raise RuntimeError( - "Create the required fields on the Baserow Parts table before " - "enabling CAD file sync: " + ", ".join(missing_fields) - ) - files_uploaded, files_cached = attach_exported_files( - client, parts, part_rows, exports_by_part, warnings - ) - for part in parts: - part["Last Synced At"] = now - part_fields = [ - "Name", - "Description", - "Material", - "Manufacturing Method", - "Vendor", - "Revision", - "OnShape Text", - "Category", - "Onshape Drawing", - "Active", - ] - if sync_cad_files: - part_fields.extend( - ( - DRAWING_PDF_FIELD, - DRAWING_PDF_KEY_FIELD, - STEP_FILE_FIELD, - STEP_KEY_FIELD, - ) - ) - upsert_table( - client, table_ids["parts"], "Part Number", parts, tuple(part_fields) - ) - part_rows = client.list_rows(table_ids["parts"]) - part_ids = {str(r.get("Part Number") or ""): r["id"] for r in part_rows} - - existing_requirements = client.list_rows(table_ids["requirements"]) - available_requirement_fields = { - field for row in existing_requirements for field in row - } - desired_requirements = [] - for requirement in requirements: - fields = { - k: v - for k, v in requirement.items() - if k not in ("part_number", "assembly_number", "_operation_machines") - } - fields["Part"] = [part_ids[requirement["part_number"]]] - fields["Assembly"] = [assembly_ids[requirement["assembly_number"]]] if requirement["assembly_number"] else [] - fields["Last Synced At"] = now - if available_requirement_fields: - fields = { - field: value - for field, value in fields.items() - if field in available_requirement_fields - } - desired_requirements.append(fields) - - source_fields = tuple( - field - for field in PRODUCTION_REQUIREMENT_MANAGED_FIELDS - if not available_requirement_fields - or field in available_requirement_fields - ) - created, updated, unchanged = upsert_table( - client, - table_ids["requirements"], - "Production Key", - desired_requirements, - source_fields, - change_flag_field="Engineering Changed", - ) - - existing_requirements = client.list_rows(table_ids["requirements"]) - desired_keys = {r["Production Key"] for r in desired_requirements} - desired_part_configurations = { - ( - fields["Part"][0], - str(fields.get("Configuration") or "default"), - ) - for fields in desired_requirements - if fields.get("Part") - } - deactivate = [] - for row in existing_requirements: - row_source_root = str(row.get("Source Root") or "").strip() - linked = row.get("Assembly") or [] - assembly_names = {str(x.get("value") or "") for x in linked if isinstance(x, dict)} - row_key = str(row.get("Production Key") or "") - if not row_source_root and row_key.count("|") >= 4: - row_source_root = row_key.split("|", 1)[0] - part_links = row.get("Part") or [] - row_part_id = next( - ( - item.get("id") - for item in part_links - if isinstance(item, dict) and item.get("id") is not None - ), - None, - ) - is_matching_legacy_row = ( - not row_source_root - and not assembly_names - and row_key.startswith("|") - and ( - row_part_id, - str(row.get("Configuration") or "default"), - ) - in desired_part_configurations - ) - is_synced_scope = row_source_root in synced_roots or ( - not row_source_root and bool(assembly_names & synced_roots) - ) or is_matching_legacy_row - if ( - is_synced_scope - and row.get("Production Key") not in desired_keys - and row.get("Active in BOM") - ): - deactivate.append({"id": row["id"], "Active in BOM": False, "Engineering Changed": True}) - if deactivate: - client.batch_update(table_ids["requirements"], deactivate) - - requirement_rows_by_key = { - str(row.get("Production Key") or ""): row - for row in client.list_rows(table_ids["requirements"]) - } - synced_requirement_ids = { - int(row["id"]) - for row in requirement_rows_by_key.values() - if str(row.get("Source Root") or "").strip() in synced_roots - } - - desired_finishing = [] - for requirement in requirements: - color = str(requirement.get("Finishing") or "None") - if color not in ("Red", "Black"): - continue - production_key = str(requirement.get("Production Key") or "") - requirement_row = requirement_rows_by_key.get(production_key) - if requirement_row is None: - raise RuntimeError( - "No Baserow Production Requirement row found for finishing " - f"queue item {production_key or '(unnamed)'}" - ) - desired_finishing.append( - { - "Production Key": production_key, - "Production Requirement": [requirement_row["id"]], - "Powder Coat Color": color, - "Required Quantity": requirement["Required Quantity"], - "Active": True, - "Last Synced At": now, - } - ) - # Machinist is assigned by manufacturing and must survive every resync. - # Finishing also has no claimed/completed quantity fields: each action - # represents the full Required Quantity for the Production Requirement. - finishing_fields = ( - "Production Requirement", - "Powder Coat Color", - "Required Quantity", - "Active", - "Last Synced At", - ) - ( - finishing_created, - finishing_updated, - finishing_unchanged, - ) = upsert_table( - client, - table_ids["finishing"], - "Production Key", - desired_finishing, - finishing_fields, - ) - desired_finishing_keys = { - row["Production Key"] for row in desired_finishing - } - deactivate_finishing = [ - {"id": row["id"], "Active": False, "Last Synced At": now} - for row in client.list_rows(table_ids["finishing"]) - if str(row.get("Production Key") or "") not in desired_finishing_keys - and row.get("Active") is not False - and bool( - linked_row_ids(row.get("Production Requirement")) - & synced_requirement_ids - ) - ] - if deactivate_finishing: - client.batch_update(table_ids["finishing"], deactivate_finishing) - - existing_operations = client.list_rows(table_ids["operations"]) - operation_statuses = operation_statuses_for_routes( - operations, existing_operations - ) - desired_operations = [] - for operation in operations: - production_key = str(operation.get("production_key") or "") - requirement_row = requirement_rows_by_key.get(production_key) - if requirement_row is None: - raise RuntimeError( - f"No Baserow Production Requirement row found for operation " - f"{operation.get('Operation') or '(unnamed)'}" - ) - desired_operations.append( - { - key: value - for key, value in { - **operation, - "Production Requirement": [requirement_row["id"]], - "Status": operation_statuses[ - str(operation.get("Operation") or "") - ], - }.items() - if key != "production_key" - } - ) - - operation_fields = ( - "Production Requirement", - "Operation Number", - "Machine", - "Status", - "Active in Routing", - ) - operations_created, operations_updated, operations_unchanged = upsert_table( - client, - table_ids["operations"], - "Operation", - desired_operations, - operation_fields, - ) - desired_operation_keys = { - operation["Operation"] for operation in desired_operations - } - deactivate_operations = [ - {"id": row["id"], "Active in Routing": False} - for row in client.list_rows(table_ids["operations"]) - if row.get("Operation") not in desired_operation_keys - and row.get("Active in Routing") is not False - and bool( - linked_row_ids(row.get("Production Requirement")) - & synced_requirement_ids - ) - ] - if deactivate_operations: - client.batch_update(table_ids["operations"], deactivate_operations) - - # Record the root revision only after the dependent tables succeed. This - # value is the next run's early-exit marker, so writing it earlier could - # hide a partial failure and prevent a retry. - if root_assembly_fields: - upsert_table( - client, - table_ids["assemblies"], - "Assembly Number", - supported_assembly_records, - root_assembly_fields, - ) - - summary = { - "created": created, - "updated": updated, - "unchanged": unchanged, - "deactivated": len(deactivate), - "roots_missing_from_master": len(missing_from_master), - "file_groups_uploaded": files_uploaded, - "file_groups_cached": files_cached, - "operations_created": operations_created, - "operations_updated": operations_updated, - "operations_unchanged": operations_unchanged, - "operations_deactivated": len(deactivate_operations), - "finishing_created": finishing_created, - "finishing_updated": finishing_updated, - "finishing_unchanged": finishing_unchanged, - "finishing_deactivated": len(deactivate_finishing), - } - warnings = sorted(set(warnings)) - client.update_one(table_ids["sync"], run["id"], { - "Finished At": utc_now(), - "Result": "Partial" if warnings else "Success", - "Requirements Created": created, - "Requirements Updated": updated, - "Requirements Unchanged": unchanged, - "Requirements Deactivated": len(deactivate), - "Warnings": "\n".join(warnings), - "GitHub Run URL": os.environ.get("GITHUB_RUN_URL", ""), - }) - return summary - except Exception as exc: - try: - client.update_one(table_ids["sync"], run["id"], {"Finished At": utc_now(), "Result": "Failed", "Error": str(exc)[:10000]}) - finally: - raise - - def environment_flag(name: str) -> bool: value = os.environ.get(name, "").strip().lower() if value in ("", "0", "false", "no", "off"): @@ -2407,7 +1924,10 @@ def run_sync( sync_cad_files: bool = False, master_target: OnshapeTarget | None = None, discover_from_master: bool = False, + run_id: str | None = None, ) -> dict: + if output_json and not dry_run: + raise ValueError("--output-json is only available with --dry-run") targets = target if isinstance(target, list) else [target] if not targets: raise ValueError("At least one manufacturing-root assembly URL is required") @@ -2422,6 +1942,10 @@ def run_sync( master_workspace_items: list[dict] = [] root_sources: list[tuple[OnshapeDocumentReference, ReleasedAssembly]] = [] document_metadata_cache: dict[str, dict | None] = {} + bulk_part_metadata_cache: dict[tuple, dict[str, dict]] = {} + single_part_metadata_cache: dict[tuple, dict | None] = {} + drawing_revision_cache: dict[tuple[str, str], dict] = {} + discovery_complete = True if discover_from_master: if len(targets) != 1: @@ -2433,16 +1957,17 @@ def run_sync( ) discovery_master_url = onshape_target_url(discovery_target) master_workspace_items = fetch_bom( - discovery_target, generate_if_absent=True + discovery_target, generate_if_absent=not dry_run ) root_sources, discovery_warnings = discover_released_manufacturing_roots( discovery_target, master_workspace_items ) warning_items.extend(discovery_warnings) + discovery_complete = not discovery_warnings if not root_sources: raise RuntimeError( "No released direct-child manufacturing roots were discovered; " - "Baserow was not changed" + "Supabase was not changed" ) master_target = None else: @@ -2453,7 +1978,7 @@ def run_sync( warning = ( "Manufacturing root " f"{onshape_target_url(root_target)} could not be resolved and " - "was skipped; existing Baserow requirements were left " + "was skipped; existing Supabase requirements were left " f"unchanged. {type(exc).__name__}: {exc}" ) warning_items.append(warning) @@ -2473,7 +1998,7 @@ def run_sync( if not root_sources: raise RuntimeError( "No configured manufacturing roots could be resolved; " - "Baserow was not changed" + "Supabase was not changed" ) resolved_root_numbers: set[str] = set() @@ -2483,6 +2008,8 @@ def run_sync( raise RuntimeError( "Released manufacturing-root assembly has no Part number" ) + if not released.revision.strip(): + raise RuntimeError(f"Manufacturing root {source_root} has no released revision") normalized_root = normalized_part_number(source_root) if normalized_root in resolved_root_numbers: raise ValueError( @@ -2490,19 +2017,38 @@ def run_sync( ) resolved_root_numbers.add(normalized_root) - if not dry_run and all_root_revisions_are_current( - [released for _, released in root_sources], discovery_master_url - ): - result = { - "skipped": True, - "reason": "All manufacturing-root revisions are already current", - "roots_checked": len(root_sources), - "source_revisions": [ - released.as_dict() for _, released in root_sources - ], - } - print(json.dumps(result, indent=2)) - return result + if not dry_run: + resolved_root_sources = list(root_sources) + stale_roots, membership_changed = stale_root_revisions( + [released for _, released in resolved_root_sources], + discovery_master_url, + sync_cad_files=sync_cad_files, + ) + if not stale_roots and not membership_changed: + result = { + "skipped": True, + "reason": "All manufacturing-root revisions are already current", + "roots_checked": len(resolved_root_sources), + "warnings": sorted(set(warning_items)), + "source_revisions": [ + released.as_dict() for _, released in resolved_root_sources + ], + } + print(json.dumps(result, indent=2)) + return result + root_sources = [source for source in resolved_root_sources + if normalized_part_number(source[1].part_number) in stale_roots] + if not root_sources: + # Membership bookkeeping has no successful BOM scope and must never + # deactivate requirements or rescan unchanged drawings. + return sync_to_supabase( + [], [], warning_items, 0, {}, False, synced_roots=set(), + discovery_master=discovery_master_url, + discovered_roots={r.part_number for _, r in resolved_root_sources}, + discovery_complete=discovery_complete, run_id=run_id, + ) + print(f"Incremental sync: processing {len(root_sources)} changed root(s) " + f"of {len(resolved_root_sources)} resolved") for root_reference, released in root_sources: source_root = released.part_number.strip() @@ -2512,50 +2058,64 @@ def run_sync( ) seen_roots.add(source_root) - released_target = released.bom_target(root_reference.base_url) - raw_items = fetch_bom(released_target) - raw_items = hydrate_operation_properties( - raw_items, prefixes, root_reference.base_url - ) - source_document_names, document_warnings = source_document_names_for_rows( - raw_items, - prefixes, - root_reference.base_url, - document_metadata_cache, - ) - root_parts, root_requirements, root_warnings = build_records( - raw_items, - prefixes, - source_root=source_root, - source_revision=released.revision, - source_document_names=source_document_names, - ) - drawing_urls, drawing_warnings = drawing_urls_for_parts( - raw_items, - prefixes, - root_reference.base_url, - [ - OnshapeDocumentReference( - released_target.base_url.rstrip("/"), - released_target.did, - released_target.wvm_type, - released_target.wvm_id, - ) - ], - ) - for part in root_parts: - part["Onshape Drawing"] = drawing_urls.get(part["Part Number"], "") - - root_exports: dict[str, list[FileExport]] = {} - export_warnings: list[str] = [] - if sync_cad_files: - root_exports, export_warnings = build_file_exports( - root_parts, + try: + released_target = released.bom_target(root_reference.base_url) + raw_items = fetch_bom(released_target) + raw_items = hydrate_operation_properties( raw_items, - drawing_urls, prefixes, root_reference.base_url, + bulk_part_metadata_cache, + single_part_metadata_cache, + ) + source_document_names, document_warnings = source_document_names_for_rows( + raw_items, + prefixes, + root_reference.base_url, + document_metadata_cache, + ) + root_parts, root_requirements, root_warnings = build_records( + raw_items, + prefixes, + source_root=source_root, + source_revision=released.revision, + source_document_names=source_document_names, ) + drawing_urls, drawing_warnings = drawing_urls_for_parts( + raw_items, + prefixes, + root_reference.base_url, + [ + OnshapeDocumentReference( + released_target.base_url.rstrip("/"), + released_target.did, + released_target.wvm_type, + released_target.wvm_id, + ) + ], + drawing_revision_cache, + ) + for part in root_parts: + part["Onshape Drawing"] = drawing_urls.get(part["Part Number"], "") + + root_exports: dict[str, list[FileExport]] = {} + export_warnings: list[str] = [] + if sync_cad_files: + root_exports, export_warnings = build_file_exports( + root_parts, + raw_items, + drawing_urls, + prefixes, + root_reference.base_url, + ) + except Exception as exc: + warning_items.append( + f"Manufacturing root {source_root} could not be synced and was skipped; " + f"existing Supabase requirements were left unchanged. {type(exc).__name__}: {exc}" + ) + seen_roots.remove(source_root) + discovery_complete = False + continue root_results.append( { "reference": root_reference, @@ -2576,6 +2136,9 @@ def run_sync( + export_warnings ) + if not root_results: + raise RuntimeError("No manufacturing roots could be synced; Supabase was not changed") + master_released: ReleasedAssembly | None = None master_items: list[dict] = [] master_revisions: dict[str, str] = {} @@ -2691,7 +2254,7 @@ def run_sync( }, "warnings": warnings, } - print("DRY RUN: no Baserow API calls were made") + print("DRY RUN: no Supabase API calls were made") if output_json: destination = Path(output_json) destination.parent.mkdir(parents=True, exist_ok=True) @@ -2703,7 +2266,7 @@ def run_sync( if output_json: raise ValueError("--output-json is only available with --dry-run") - summary = sync_to_baserow( + summary = sync_to_supabase( parts, requirements, warnings, @@ -2714,6 +2277,9 @@ def run_sync( assembly_records=assembly_records, synced_roots=seen_roots, discovery_master=discovery_master_url, + discovered_roots={released.part_number for _, released in resolved_root_sources} if not dry_run else seen_roots, + discovery_complete=discovery_complete, + run_id=run_id, ) print(json.dumps(summary, indent=2)) return summary @@ -2725,7 +2291,7 @@ def main(argv: list[str] | None = None) -> int: "--dry-run", action="store_true", default=environment_flag("DRY_RUN"), - help="resolve the released BOM and build records without calling Baserow", + help="resolve the released BOM and build records without calling Supabase", ) parser.add_argument( "--output-json", @@ -2747,14 +2313,32 @@ def main(argv: list[str] | None = None) -> int: for p in os.environ.get("PARTNUMBER_PREFIXES", "").split(",") if p.strip() ] - run_sync( - sync_targets, - prefixes, - dry_run=args.dry_run, - output_json=args.output_json, - sync_cad_files=environment_flag("SYNC_CAD_FILES"), - discover_from_master=not use_subassembly_list, - ) + if args.output_json and not args.dry_run: + parser.error("--output-json is only available with --dry-run") + reset_onshape_call_counts() + client = None if args.dry_run else SupabaseClient.from_env() + run_id = client.begin_run() if client else None + try: + result = run_sync( + sync_targets, + prefixes, + dry_run=args.dry_run, + output_json=args.output_json, + sync_cad_files=environment_flag("SYNC_CAD_FILES"), + discover_from_master=not use_subassembly_list, + run_id=run_id, + ) + if client and result.get("skipped"): + client.finish_run(run_id, "partial" if result.get("warnings") else "success", result) + except Exception as exc: + if client: + try: + client.finish_run(run_id, "failed", {"error": str(exc)[:10000]}) + except Exception as audit_error: + print(f"WARNING: Could not record sync failure: {audit_error}") + raise + finally: + print("Onshape API calls: " + json.dumps(onshape_call_summary(), sort_keys=True)) return 0 diff --git a/pre-2027-onshape_ci/SUPABASE_SETUP.md b/pre-2027-onshape_ci/SUPABASE_SETUP.md new file mode 100644 index 0000000..7b5dc67 --- /dev/null +++ b/pre-2027-onshape_ci/SUPABASE_SETUP.md @@ -0,0 +1,156 @@ +# Onshape → Supabase engineering sync + +## Contract and deployment + +The authoritative contract is `coreylu2027/190-Manufacturing`, branch +`experimental`, commit `b915b1439ae4fed616f24935de552734ebea525b`, specifically: + +- `supabase/production/20260905_normalized_manufacturing.sql` +- `supabase/production/20260905_manufacturing_attachments.sql` +- `lib/manufacturing/model.ts` + +The existing API offers bounded reads, shop writes and immutable attachment +registration. It does not offer transactional engineering synchronization or +attachment replacement. The prepared migration is +[`20260906_onshape_engineering_sync.sql`](../supabase/production/20260906_onshape_engineering_sync.sql). +It has **not been applied**. Apply it only after explicit backend approval, after +the normalized schema and attachment schema exist. It requires the existing +`manufacturing-files` bucket to be private; it neither creates nor reconfigures +that bucket. The contract's nonunique operation index is retained. Compatible +duplicate operation rows receive the same engineering fields while keeping each +row's shop work and allocations; the result includes a warning. A duplicate key +pointing at a different requirement or work type rolls the entire sync back. +The migration never merges or deletes existing shop work. + +The migration adds internal assembly revision/discovery/CAD-completion columns, +an audit table and five `public` RPCs. They use `security definer` with an empty +search path and fully qualified tables. Only `service_role` receives execute +permission. Existing table grants, RLS and PostgREST schema exposure are unchanged. +New attachments have no legacy source field ID, so `source_field_id` becomes +nullable; existing imported identifiers remain intact. The catalog and private +file RPCs remain compatible with the authoritative application model. + +## Configuration + +Production GitHub Actions secrets: + +| Secret | Purpose | +| --- | --- | +| `NEXT_PUBLIC_SUPABASE_URL` | Project HTTPS URL | +| `SUPABASE_SECRET_KEY` | Server-side Supabase secret key | +| `ONSHAPE_ACCESS_KEY` | Existing Onshape access key | +| `ONSHAPE_SECRET_KEY` | Existing Onshape signing secret | +| `ONSHAPE_DOC_URL_EPSILON` | Default Poot Horse discovery Main URL | +| `ONSHAPE_DOC_URL_DELTA` | Default Delta discovery Main URL | + +The client follows the application's authentication contract: secret keys use +`apikey`; legacy service-role JWTs also use `Authorization: Bearer`. See +[Supabase API key documentation](https://supabase.com/docs/guides/getting-started/api-keys). +Keys are never written to files or dry-run output. HTTP redirects are refused. +No table IDs or destination schema configuration are required. + +The Onshape controls remain `USE_SUBASSEMBLY_LIST`, +`ONSHAPE_SUBASSEMBLY_URLS`, `ONSHAPE_DOC_URL`, `PARTNUMBER_PREFIXES`, +`SYNC_CAD_FILES`, and `ONSHAPE_EXPORT_TIMEOUT_SECONDS`. Workflows retain the +existing triggers, URL defaults, branch guards and dry-run isolation, with +Supabase names and dispatch types. Updating actual GitHub secrets or external +dispatchers is a separate deployment action; this change updates their references. + +Install `requirements-sync.txt` for this sync. The original `requirements.txt` +continues serving the independent Sheets scripts. + +## Ownership and atomicity + +The client submits business keys, never database IDs. The RPC resolves assemblies +by `assembly_number`, parts by `part_number`, and dependent work by +`production_key`. It upserts the explicit engineering allowlist from the model, +including BOM-required quantities. It rejects shop fields and updates no shop +status, assignment, QC, location, claimed/completed quantity, ledger, CAM notes, +or allocation records. New rows leave shop columns at their schema defaults; +the manufacturing application projects queue readiness. `onshape_url` receives +the source URL, rather than the Onshape lifecycle-state label. + +Only fully processed roots enter `synced_roots`. Their stale requirements, +operations and finishing rows deactivate together; failed, unresolved and +unchanged roots retain their work. Legacy requirement scope is accepted only +from an explicit source root, a root-bearing production key or a linked root +assembly. Ambiguous unscoped rows are left alone rather than guessing from a +shared part. A valid empty released BOM can deactivate its root's old work. + +Discovery membership uses the full resolved root list, separately from changed +roots. Missing roots receive `Missing from Main — Review`; their requirements +are not deactivated. Incomplete discovery suppresses missing-root updates. +Membership-only runs do not fetch released BOMs or drawings. + +The CLI starts an audit run before Onshape resolution. Resolution failures are +recorded as failed. Successful roots and warnings commit as a partial result; +clean runs commit as success. A PL/pgSQL exception block rolls back **all** BOM, +root-marker and attachment changes before recording a failed result. A completed +run ID returns its existing result on retry. Engineering commits are serialized; +a run that started before a newer engineering commit is rejected and must resolve +fresh state. Shop transactions do not participate in that engineering lock. + +If the database is unreachable, no client can guarantee writing a failure audit; +the CLI reports that failure. Process cancellation may leave a `running` audit +row. No partial BOM transaction can commit in either case. + +## Files and API budget + +PDF/STEP bytes are exported only in production with CAD sync enabled. Content +hashes define private `sha256//.pdf` or `.step` storage paths. +Uploads never overwrite objects; duplicates are accepted only after downloading +and verifying the bytes. Each complete part/kind group updates its catalog slots, +file metadata and export key in the BOM transaction. Failed groups preserve the +previous files and keys. Changed groups replace catalog entries idempotently and +remove obsolete positions. Storage objects are never deleted by this script. + +Storage and PostgreSQL cannot share a transaction. Files are staged and verified +before the database commit. A failed export group or database transaction can +leave unreferenced private objects, but cannot leave a partial catalog or BOM. +Future retries reuse content-addressed objects; cleanup is a separate operation. + +Unchanged released root revisions exit before BOM, metadata or drawing scans. +Roots with no released revision never reach drawing discovery. Changed roots +share document-name, bulk part-metadata, fallback and document-revision caches. +Empty matching BOMs issue no drawing requests. PDF and STEP export keys skip +unchanged translations. Failed export groups leave the CAD-completion marker +false so the next CAD-enabled run retries. Enabling CAD after a BOM-only sync +also triggers a pass. Ordinary drawing releases are reconsidered when their +manufacturing root revision changes, not continuously polled independently. + +## Dry runs and offline validation + +`--dry-run` needs no **Supabase** credentials and makes no destination requests. +It still needs Onshape read credentials; running it against Onshape requires +approval. It never requests BOM generation, starts translations, or uploads +files. `--output-json PATH` is valid only with `--dry-run`. + +The following tests need no backend credentials and contact neither backend: + +```sh +python -m unittest discover -s pre-2027-onshape_ci -p test_onshape_to_supabase.py +npm install --no-save --package-lock=false @electric-sql/pglite@0.3.14 +node tests/engineering-sync.test.mjs +node tests/imported-sync.test.mjs +``` + +The Python suite blocks real HTTP calls. The PostgreSQL suite runs in memory, +loads exact contract fixtures, and tests role permissions, all shop-owned fields, +business-key relationships, scoped deactivation, late-error rollback, private +attachments, retries and overlapping engineering runs. The tests workflow has +no backend secrets. Installing the test runtime uses the package registry only. + +## Imported-data preflight + +A read-only inspection of the existing manufacturing project found 63 duplicate +operation keys (126 imported, active rows). Eight of those rows have shop activity +and allocations. Assembly latest-release markers and requirement source-root / +source-revision columns were empty. A new hosted branch was not created; local +regressions reproduce these data shapes with synthetic values. + +Even if Onshape releases have not changed, the initial sync cannot use absent +revision markers as proof of a complete prior sync. Its first approved Onshape +pass must reconcile the engineering baseline. Matching business keys update +existing rows, preserving duplicate operations and shop work. Only then are +revision markers available for subsequent early exits. No Onshape calls or +production mutations were used in this preflight. diff --git a/pre-2027-onshape_ci/__pycache__/OnshapeToBaserow.cpython-314.pyc b/pre-2027-onshape_ci/__pycache__/OnshapeToBaserow.cpython-314.pyc deleted file mode 100644 index 4b3775e..0000000 Binary files a/pre-2027-onshape_ci/__pycache__/OnshapeToBaserow.cpython-314.pyc and /dev/null differ diff --git a/pre-2027-onshape_ci/__pycache__/test_onshape_to_baserow.cpython-314.pyc b/pre-2027-onshape_ci/__pycache__/test_onshape_to_baserow.cpython-314.pyc deleted file mode 100644 index ea04802..0000000 Binary files a/pre-2027-onshape_ci/__pycache__/test_onshape_to_baserow.cpython-314.pyc and /dev/null differ diff --git a/pre-2027-onshape_ci/requirements-sync.txt b/pre-2027-onshape_ci/requirements-sync.txt new file mode 100644 index 0000000..f1995ac --- /dev/null +++ b/pre-2027-onshape_ci/requirements-sync.txt @@ -0,0 +1 @@ +requests>=2.32.3,<3 diff --git a/pre-2027-onshape_ci/test_onshape_to_baserow.py b/pre-2027-onshape_ci/test_onshape_to_supabase.py similarity index 60% rename from pre-2027-onshape_ci/test_onshape_to_baserow.py rename to pre-2027-onshape_ci/test_onshape_to_supabase.py index 76208a1..b673f7c 100644 --- a/pre-2027-onshape_ci/test_onshape_to_baserow.py +++ b/pre-2027-onshape_ci/test_onshape_to_supabase.py @@ -7,17 +7,26 @@ import tempfile import types import unittest -from unittest.mock import patch +from unittest.mock import patch, Mock -MODULE_PATH = Path(__file__).with_name("OnshapeToBaserow.py") +MODULE_PATH = Path(__file__).with_name("OnshapeToSupabase.py") sys.modules.setdefault("requests", types.ModuleType("requests")) -SPEC = importlib.util.spec_from_file_location("onshape_to_baserow", MODULE_PATH) +SPEC = importlib.util.spec_from_file_location("onshape_to_supabase", MODULE_PATH) MODULE = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = MODULE SPEC.loader.exec_module(MODULE) +def denied_network(*args, **kwargs): + raise AssertionError("Tests must not call live APIs") + + +MODULE.requests.get = denied_network +MODULE.requests.post = denied_network +MODULE.requests.Session = denied_network + + def source(url, indent=1): return {"viewHref": url, "indentLevel": indent} @@ -177,6 +186,47 @@ def drawing_revision(part_number, document_id, version_id, element_id, **overrid return item +class OnshapeCallTelemetryTests(unittest.TestCase): + def tearDown(self): + MODULE.reset_onshape_call_counts() + + def test_request_wrapper_counts_calls_by_endpoint_category(self): + MODULE.reset_onshape_call_counts() + document_id = "1" * 24 + url = f"https://cad.onshape.com/api/v16/revisions/d/{document_id}" + + with patch.object( + MODULE.requests, + "get", + return_value=FakeResponse({"items": []}), + create=True, + ), patch.object(MODULE, "onshape_headers", return_value={}): + MODULE.onshape_get_json(url) + + MODULE.record_onshape_call( + "GET", + f"https://cad.onshape.com/api/v16/metadata/d/{document_id}/v/" + f"{'2' * 24}/e/{'3' * 24}/p?thumbnail=false", + ) + MODULE.record_onshape_call( + "GET", + f"https://cad.onshape.com/api/v16/assemblies/d/{document_id}/v/" + f"{'2' * 24}/e/{'3' * 24}/bom", + ) + + self.assertEqual( + MODULE.onshape_call_summary(), + { + "total": 3, + "by_category": { + "bom": 1, + "document_revisions": 1, + "part_metadata_bulk": 1, + }, + }, + ) + + class ReleaseResolutionTests(unittest.TestCase): def test_document_url_preserves_configuration(self): parsed = MODULE.parse_onshape_doc_url( @@ -214,7 +264,7 @@ def test_main_uses_subassembly_list_without_master_discovery(self): with patch.dict(os.environ, environment, clear=True), patch.object( MODULE, "run_sync" ) as run_sync: - result = MODULE.main([]) + result = MODULE.main(["--dry-run"]) self.assertEqual(result, 0) targets = run_sync.call_args.args[0] @@ -233,7 +283,7 @@ def test_main_can_opt_out_to_master_discovery(self): with patch.dict(os.environ, environment, clear=True), patch.object( MODULE, "run_sync" ) as run_sync: - result = MODULE.main([]) + result = MODULE.main(["--dry-run"]) self.assertEqual(result, 0) self.assertEqual(run_sync.call_args.args[0].did, DID) @@ -465,13 +515,15 @@ def test_v16_dry_run_json_contains_matching_parts_and_requirements(self): [], ), ), patch.object( - MODULE, "fetch_part_metadata", return_value={"properties": []} + MODULE, + "fetch_parts_metadata", + return_value={"items": [{"partId": "JHD", "properties": []}]}, ), patch.object( MODULE, "fetch_document_metadata", return_value={"name": "A-26C-0001"}, ), patch.object( - MODULE, "sync_to_baserow", side_effect=AssertionError("Baserow called") + MODULE, "sync_to_supabase", side_effect=AssertionError("Supabase called") ): MODULE.run_sync( target(), @@ -488,7 +540,8 @@ def test_v16_dry_run_json_contains_matching_parts_and_requirements(self): ["P-190B-260100"], ) self.assertEqual(saved["parts"][0]["Revision"], "C") - self.assertEqual(saved["parts"][0]["OnShape Text"], "RELEASED") + self.assertTrue(saved["parts"][0]["OnShape Text"].startswith( + "https://cad.onshape.com/documents/part/v/version/e/studio")) self.assertEqual(saved["parts"][0]["Material"], "Aluminum - 6061") self.assertEqual( saved["parts"][0]["Onshape Drawing"], @@ -518,7 +571,7 @@ def test_v16_dry_run_json_contains_matching_parts_and_requirements(self): {MODULE.DRAWING_PDF_FIELD, MODULE.STEP_FILE_FIELD}, ) - def test_dry_run_writes_records_without_baserow(self): + def test_dry_run_writes_records_without_supabase(self): released = MODULE.released_assembly_from_revision(revision("B", VID_B)) rows = [ {"name": "A-190B-260001", "partNumber": "", "itemSource": source("", 0)}, @@ -539,7 +592,7 @@ def test_dry_run_writes_records_without_baserow(self): ), patch.object(MODULE, "fetch_bom", return_value=rows), patch.object( MODULE, "drawing_urls_for_parts", return_value=({}, []) ), patch.object( - MODULE, "sync_to_baserow", side_effect=AssertionError("Baserow called") + MODULE, "sync_to_supabase", side_effect=AssertionError("Supabase called") ): result = MODULE.run_sync( target(), ["P-190B-26"], dry_run=True, output_json=str(output) @@ -689,13 +742,9 @@ def test_missing_source_and_unavailable_metadata_warn_without_failing(self): class DrawingLinkTests(unittest.TestCase): - def test_discovery_and_release_lookup_are_cached_per_document(self): + def test_document_revisions_are_cached_and_newest_drawing_is_selected(self): part_did = "1" * 24 part_vid = "2" * 24 - drawing_eid = "4" * 24 - released_did = "6" * 24 - released_vid = "7" * 24 - released_eid = "8" * 24 item_source = { "documentId": part_did, "wvmType": "v", @@ -715,124 +764,44 @@ def test_discovery_and_release_lookup_are_cached_per_document(self): "itemSource": {**item_source, "configuration": "Length=2+inch"}, }, ] - elements = [ - {"id": "5" * 24, "name": "Part Studio 1", "elementType": "PARTSTUDIO"}, - {"id": drawing_eid, "name": "p-190b-260100", "elementType": "DRAWING"}, - ] - + older = drawing_revision( + "P-190B-260100", part_did, "6" * 24, "7" * 24, + releaseCreatedDate="2026-01-01T00:00:00Z", + ) + newest = drawing_revision( + "P-190B-260100", part_did, "8" * 24, "9" * 24, + releaseCreatedDate="2026-02-01T00:00:00Z", + ) with patch.object( - MODULE, "fetch_document_elements", return_value=elements - ) as fetch_elements, patch.object( - MODULE, - "fetch_latest_drawing_revision", - return_value=drawing_revision( - "P-190B-260100", released_did, released_vid, released_eid - ), - ) as fetch_revision: + MODULE, "fetch_document_revisions", return_value={"items": [newest, older]} + ) as fetch_revisions: drawing_urls, warnings = MODULE.drawing_urls_for_parts( rows, ["P-190B-26"], "https://cad.onshape.com" ) - self.assertEqual(fetch_elements.call_count, 1) - self.assertEqual(fetch_revision.call_count, 1) + self.assertEqual(fetch_revisions.call_count, 1) self.assertEqual(warnings, []) self.assertEqual( drawing_urls["P-190B-260100"], - f"https://cad.onshape.com/documents/{released_did}/v/{released_vid}/e/" - f"{released_eid}", + f"https://cad.onshape.com/documents/{part_did}/v/{'8' * 24}/e/" + f"{'9' * 24}", ) - def test_drawing_metadata_part_number_matches_when_tab_name_does_not(self): - part_did = "1" * 24 - part_vid = "2" * 24 - rows = [ - { - "partNumber": "P-190B-260100", - "itemSource": { - "documentId": part_did, - "wvmType": "v", - "wvmId": part_vid, - }, - } - ] - elements = [ - { - "id": "4" * 24, - "name": "Right Support Plate Drawing 1", - "elementType": "APPLICATION", - "mimeType": "application/vnd.onshape.drawing", - } - ] - metadata = { - "properties": [ - {"name": "Name", "value": "Right Support Plate Drawing 1"}, - {"name": "Part number", "value": "P-190B-260100"}, - ] - } - - with patch.object( - MODULE, "fetch_document_elements", return_value=elements - ), patch.object( - MODULE, "fetch_element_metadata", return_value=metadata - ) as fetch_metadata, patch.object( - MODULE, - "fetch_latest_drawing_revision", - return_value=drawing_revision( - "P-190B-260100", "5" * 24, "6" * 24, "7" * 24 - ), - ): - drawing_urls, warnings = MODULE.drawing_urls_for_parts( - rows, ["P-190B-26"], "https://frc190.onshape.com" - ) - - self.assertIn("P-190B-260100", drawing_urls) - self.assertEqual(warnings, []) - fetch_metadata.assert_called_once_with( - MODULE.OnshapeDocumentReference( - "https://frc190.onshape.com", part_did, "v", part_vid - ), - "4" * 24, - ) - - def test_released_assembly_document_is_also_scanned_for_drawings(self): + def test_released_assembly_document_is_included_as_drawing_source(self): released_reference = MODULE.OnshapeDocumentReference( "https://frc190.onshape.com", "3" * 24, "v", "4" * 24 ) - drawing_eid = "5" * 24 rows = [ { "partNumber": "P-190B-260764", "itemSource": None, } ] - - def elements_for(reference): - if reference == released_reference: - return [ - { - "id": drawing_eid, - "name": "Right Support Plate Drawing 1", - "elementType": "APPLICATION", - } - ] - return [] - + latest = drawing_revision( + "P-190B-260764", "6" * 24, "7" * 24, "8" * 24 + ) with patch.object( - MODULE, "fetch_document_elements", side_effect=elements_for - ), patch.object( - MODULE, - "fetch_element_metadata", - return_value={ - "properties": [ - {"name": "Part number", "value": "P-190B-260764"} - ] - }, - ), patch.object( - MODULE, - "fetch_latest_drawing_revision", - return_value=drawing_revision( - "P-190B-260764", "6" * 24, "7" * 24, "8" * 24 - ), + MODULE, "fetch_document_revisions", return_value={"items": [latest]} ): drawing_urls, warnings = MODULE.drawing_urls_for_parts( rows, @@ -869,91 +838,64 @@ def test_multiple_matching_drawings_warn_and_leave_link_blank(self): }, }, ] - elements = [ - {"id": "5" * 24, "name": "P-190B-260100", "elementType": "DRAWING"}, - ] - - def latest_for(reference, part_number): - return drawing_revision( - part_number, - reference.did, - "6" * 24 if reference.did == first_did else "7" * 24, - "8" * 24 if reference.did == first_did else "9" * 24, - ) + def revisions_for(base_url, document_id): + return { + "items": [ + drawing_revision( + "P-190B-260100", + document_id, + "6" * 24 if document_id == first_did else "7" * 24, + "8" * 24 if document_id == first_did else "9" * 24, + ) + ] + } with patch.object( - MODULE, "fetch_document_elements", return_value=elements - ), patch.object( - MODULE, "fetch_latest_drawing_revision", side_effect=latest_for - ): + MODULE, "fetch_document_revisions", side_effect=revisions_for + ) as fetch_revisions: drawing_urls, warnings = MODULE.drawing_urls_for_parts( rows, ["P-190B-26"], "https://cad.onshape.com" ) + self.assertEqual(fetch_revisions.call_count, 2) self.assertNotIn("P-190B-260100", drawing_urls) self.assertEqual(len(warnings), 1) self.assertIn("Multiple released drawings", warnings[0]) - def test_workspace_and_assembly_snapshots_resolve_to_drawing_own_release(self): - workspace_reference = MODULE.OnshapeDocumentReference( - "https://frc190.onshape.com", "1" * 24, "w", "2" * 24 - ) - released_reference = MODULE.OnshapeDocumentReference( - "https://frc190.onshape.com", "1" * 24, "v", "3" * 24 - ) - drawing_eid = "4" * 24 + def test_revision_cache_is_shared_across_root_calls(self): + document_id = "1" * 24 rows = [ { "partNumber": "P-190B-260764", "itemSource": { - "documentId": workspace_reference.did, - "wvmType": workspace_reference.wvm_type, - "wvmId": workspace_reference.wvm_id, + "documentId": document_id, + "wvmType": "v", + "wvmId": "2" * 24, }, } ] - elements = [ - { - "id": drawing_eid, - "name": "Right Support Plate Drawing 1", - "elementType": "APPLICATION", - } - ] - - own_did = "5" * 24 - own_vid = "6" * 24 - own_eid = "7" * 24 + payload = { + "items": [ + drawing_revision( + "P-190B-260764", document_id, "6" * 24, "7" * 24 + ) + ] + } + cache = {} with patch.object( - MODULE, "fetch_document_elements", return_value=elements - ), patch.object( - MODULE, - "fetch_element_metadata", - return_value={ - "properties": [ - {"name": "Part number", "value": "P-190B-260764"} - ] - }, - ), patch.object( - MODULE, - "fetch_latest_drawing_revision", - return_value=drawing_revision( - "P-190B-260764", own_did, own_vid, own_eid - ), - ) as fetch_revision: - drawing_urls, warnings = MODULE.drawing_urls_for_parts( - rows, - ["P-190B-26"], - "https://frc190.onshape.com", - [released_reference], + MODULE, "fetch_document_revisions", return_value=payload + ) as fetch_revisions: + first = MODULE.drawing_urls_for_parts( + rows, ["P-190B-26"], "https://frc190.onshape.com", + revision_cache=cache, + ) + second = MODULE.drawing_urls_for_parts( + rows, ["P-190B-26"], "https://frc190.onshape.com", + revision_cache=cache, ) - self.assertEqual(warnings, []) - self.assertEqual(fetch_revision.call_count, 1) - self.assertEqual( - drawing_urls["P-190B-260764"], - f"https://frc190.onshape.com/documents/{own_did}/" - f"v/{own_vid}/e/{own_eid}", - ) + self.assertEqual(fetch_revisions.call_count, 1) + self.assertEqual(first, second) def test_unreleased_drawing_is_not_used_as_pdf_source(self): part_number = "P-190B-260100" @@ -967,13 +909,9 @@ def test_unreleased_drawing_is_not_used_as_pdf_source(self): }, } ] - elements = [ - {"id": "3" * 24, "name": part_number, "elementType": "DRAWING"} - ] - with patch.object( - MODULE, "fetch_document_elements", return_value=elements - ), patch.object(MODULE, "fetch_latest_drawing_revision", return_value=None): + MODULE, "fetch_document_revisions", return_value={"items": []} + ): drawing_urls, warnings = MODULE.drawing_urls_for_parts( rows, ["P-190B-26"], "https://cad.onshape.com" ) @@ -982,31 +920,12 @@ def test_unreleased_drawing_is_not_used_as_pdf_source(self): self.assertEqual(len(warnings), 1) self.assertIn("No released drawing revision", warnings[0]) - def test_latest_drawing_revision_uses_drawing_type_and_handles_204(self): - reference = MODULE.OnshapeDocumentReference( - "https://frc190.onshape.com", "1" * 24, "v", "2" * 24 - ) - with patch.object( - MODULE.requests, "get", return_value=FakeResponse(None, 204), create=True - ) as get, patch.object(MODULE, "onshape_headers", return_value={}): - latest = MODULE.fetch_latest_drawing_revision( - reference, "P-190B-260100" - ) - - self.assertIsNone(latest) - self.assertIn("/p/P-190B-260100/latest?et=2", get.call_args.args[0]) - class FileExportTests(unittest.TestCase): def sample_export(self, field=MODULE.STEP_FILE_FIELD): return MODULE.FileExport( part_number="P-190B-260100", field_name=field, - key_field_name=( - MODULE.STEP_KEY_FIELD - if field == MODULE.STEP_FILE_FIELD - else MODULE.DRAWING_PDF_KEY_FIELD - ), source_key="source-key", filename="P-190B-260100_rev-C.step", content_type="application/step", @@ -1088,120 +1007,16 @@ def test_unlisted_method_does_not_plan_or_warn_about_step_export(self): self.assertEqual(exports, {}) self.assertEqual(warnings, []) - def test_current_file_and_export_key_skip_translation(self): - export = self.sample_export() - expected_key = MODULE.aggregate_export_key([export]) - parts = [{"Part Number": export.part_number}] - existing = [ - { - "Part Number": export.part_number, - MODULE.STEP_FILE_FIELD: [{"name": "stored-step"}], - MODULE.STEP_KEY_FIELD: expected_key, - } - ] - - with patch.object( - MODULE, - "start_file_translation", - side_effect=AssertionError("translation started"), - ): - uploaded, cached = MODULE.attach_exported_files( - object(), parts, existing, {export.part_number: [export]}, [] - ) - - self.assertEqual((uploaded, cached), (0, 1)) - self.assertEqual(parts[0][MODULE.STEP_FILE_FIELD], [{"name": "stored-step"}]) - - def test_changed_export_is_downloaded_and_uploaded(self): - export = self.sample_export() - parts = [{"Part Number": export.part_number}] - warnings = [] - - class Client: - def upload_file(self, filename, content, content_type): - self.upload = (filename, content, content_type) - return {"name": "baserow-file-name"} - - client = Client() - with patch.object( - MODULE, "start_file_translation", return_value={"id": "translation"} - ), patch.object( - MODULE, - "wait_for_translation", - return_value={ - "resultExternalDataIds": ["external"], - "exportRuleFileName": "Configured Shop Export", - }, - ), patch.object(MODULE, "download_translation", return_value=b"STEP"): - uploaded, cached = MODULE.attach_exported_files( - client, parts, [], {export.part_number: [export]}, warnings - ) - - self.assertEqual((uploaded, cached), (1, 0)) - self.assertEqual(warnings, []) - self.assertEqual( - parts[0][MODULE.STEP_FILE_FIELD], [{"name": "baserow-file-name"}] - ) - self.assertEqual( - parts[0][MODULE.STEP_KEY_FIELD], MODULE.aggregate_export_key([export]) - ) - self.assertEqual(client.upload[0], "Configured Shop Export.step") - self.assertEqual(client.upload[1], b"STEP") - - def test_failed_refresh_does_not_clear_the_previous_attachment(self): - export = self.sample_export() - parts = [{"Part Number": export.part_number}] - existing = [ - { - "Part Number": export.part_number, - MODULE.STEP_FILE_FIELD: [{"name": "previous"}], - MODULE.STEP_KEY_FIELD: "old-key", - } - ] - warnings = [] - - with patch.object( - MODULE, "start_file_translation", side_effect=RuntimeError("denied") - ): - uploaded, cached = MODULE.attach_exported_files( - object(), parts, existing, {export.part_number: [export]}, warnings - ) - - self.assertEqual((uploaded, cached), (0, 0)) - self.assertEqual(parts[0][MODULE.STEP_FILE_FIELD], [{"name": "previous"}]) - self.assertEqual(parts[0][MODULE.STEP_KEY_FIELD], "old-key") - self.assertIn("Could not start STEP File export", warnings[0]) - - def test_multiple_file_values_can_be_compared(self): - existing = { - MODULE.STEP_FILE_FIELD: [ - {"name": "second", "url": "https://files/second"}, - {"name": "first", "url": "https://files/first"}, - ] - } - desired = { - MODULE.STEP_FILE_FIELD: [ - {"name": "first"}, - {"name": "second"}, - ] - } - - self.assertFalse( - MODULE.changed( - existing, desired, (MODULE.STEP_FILE_FIELD,) - ) - ) - class RecordBuildingTests(unittest.TestCase): - def test_every_baserow_machine_name_is_normalized_case_insensitively(self): - for machine in MODULE.BASEROW_MACHINE_NAMES: + def test_every_supabase_machine_name_is_normalized_case_insensitively(self): + for machine in MODULE.MACHINE_NAMES: with self.subTest(machine=machine): self.assertEqual( MODULE.operation_machine_name(machine.swapcase()), machine ) - def test_requirement_machine_fields_use_exact_baserow_choice_casing(self): + def test_requirement_machine_fields_use_exact_supabase_choice_casing(self): fields = MODULE.production_requirement_machine_fields( { "Manufacturing Method": "countersinking", @@ -1249,8 +1064,14 @@ def test_part_operation_metadata_uses_immutable_configured_source_and_cache(self } with patch.object( - MODULE, "fetch_part_metadata", return_value=metadata - ) as fetch_metadata: + MODULE, + "fetch_parts_metadata", + return_value={"items": [{"partId": "JHD", **metadata}]}, + ) as fetch_metadata, patch.object( + MODULE, + "fetch_part_metadata", + side_effect=AssertionError("single-part fallback used"), + ): hydrated = MODULE.hydrate_operation_properties( rows, ["P-190B-26"], "https://frc190.onshape.com" ) @@ -1262,6 +1083,39 @@ def test_part_operation_metadata_uses_immutable_configured_source_and_cache(self ) self.assertEqual(hydrated[0]["Powder Coat Color"], "Red") + def test_missing_bulk_part_uses_cached_single_part_fallback(self): + item_source = { + "documentId": "1" * 24, + "wvmType": "v", + "wvmId": "2" * 24, + "elementId": "3" * 24, + "partId": "JHD", + "configuration": "default", + } + rows = [ + {"partNumber": "P-190B-260100", "itemSource": item_source}, + {"partNumber": "P-190B-260100", "itemSource": item_source}, + ] + metadata = { + "properties": [ + {"name": "Manufacturing Method", "value": "HAAS CNC"} + ] + } + + with patch.object( + MODULE, "fetch_parts_metadata", return_value={"items": []} + ) as fetch_bulk, patch.object( + MODULE, "fetch_part_metadata", return_value=metadata + ) as fetch_single: + hydrated = MODULE.hydrate_operation_properties( + rows, ["P-190B-26"], "https://frc190.onshape.com" + ) + + self.assertEqual(fetch_bulk.call_count, 1) + self.assertEqual(fetch_single.call_count, 1) + self.assertEqual(hydrated[0]["Manufacturing Method"], "HAAS CNC") + self.assertEqual(hydrated[1]["Manufacturing Method"], "HAAS CNC") + def test_part_metadata_request_includes_configuration(self): item_source = { "documentId": "1" * 24, @@ -1281,6 +1135,22 @@ def test_part_metadata_request_includes_configuration(self): self.assertIn("/e/" + "3" * 24 + "/p/JHD", url) self.assertIn("configuration=Length%3D2%2Binch", url) + def test_bulk_part_metadata_request_includes_configuration(self): + reference = MODULE.OnshapeDocumentReference( + "https://frc190.onshape.com", "1" * 24, "v", "2" * 24 + ) + with patch.object( + MODULE, "onshape_get_json", return_value={"items": []} + ) as get: + MODULE.fetch_parts_metadata( + reference, "3" * 24, "Length=2+inch" + ) + + url = get.call_args.args[0] + self.assertIn("/e/" + "3" * 24 + "/p?", url) + self.assertIn("includeComputedAssemblyProperties=false", url) + self.assertIn("configuration=Length%3D2%2Binch", url) + def test_operations_use_op_labels_case_insensitive_properties_and_aliases(self): rows = [ { @@ -1332,101 +1202,6 @@ def test_operations_use_op_labels_case_insensitive_properties_and_aliases(self): ) self.assertTrue(all("OP4" not in operation["Operation"] for operation in operations)) - def test_operation_statuses_gate_each_route_on_its_predecessor(self): - operations = [ - { - "Operation": "route-a|OP2", - "production_key": "route-a", - "Operation Number": "OP2", - }, - { - "Operation": "route-a|OP1", - "production_key": "route-a", - "Operation Number": "OP1", - }, - { - "Operation": "route-b|OP3", - "production_key": "route-b", - "Operation Number": "OP3", - }, - ] - - statuses = MODULE.operation_statuses_for_routes(operations, []) - - self.assertEqual( - statuses, - { - "route-a|OP1": "Ready", - "route-a|OP2": "Planned", - "route-b|OP3": "Ready", - }, - ) - - def test_operation_statuses_unlock_next_op_and_preserve_work_states(self): - operations = [ - { - "Operation": "route-a|OP1", - "production_key": "route-a", - "Operation Number": "OP1", - }, - { - "Operation": "route-a|OP2", - "production_key": "route-a", - "Operation Number": "OP2", - }, - { - "Operation": "route-a|OP3", - "production_key": "route-a", - "Operation Number": "OP3", - }, - ] - existing_rows = [ - { - "Operation": "route-a|OP1", - "Status": {"id": 1, "value": "Complete"}, - }, - { - "Operation": "route-a|OP2", - "Status": {"id": 2, "value": "Planned"}, - }, - { - "Operation": "route-a|OP3", - "Status": {"id": 3, "value": "Blocked"}, - }, - ] - - statuses = MODULE.operation_statuses_for_routes( - operations, existing_rows - ) - - self.assertEqual(statuses["route-a|OP1"], "Complete") - self.assertEqual(statuses["route-a|OP2"], "Ready") - self.assertEqual(statuses["route-a|OP3"], "Blocked") - - def test_operation_statuses_hide_a_prematurely_ready_downstream_op(self): - operations = [ - { - "Operation": "route-a|OP1", - "production_key": "route-a", - "Operation Number": "OP1", - }, - { - "Operation": "route-a|OP2", - "production_key": "route-a", - "Operation Number": "OP2", - }, - ] - existing_rows = [ - {"Operation": "route-a|OP1", "Status": "In Progress"}, - {"Operation": "route-a|OP2", "Status": "Ready"}, - ] - - statuses = MODULE.operation_statuses_for_routes( - operations, existing_rows - ) - - self.assertEqual(statuses["route-a|OP1"], "In Progress") - self.assertEqual(statuses["route-a|OP2"], "Planned") def test_custom_bom_header_display_name_is_available_for_operations(self): normalized = MODULE.normalize_bom_rows( @@ -1508,237 +1283,6 @@ def test_bom_item_is_preserved_as_text(self): self.assertEqual(requirements[0]["BOM Positions"], "1.10") -class BaserowClientTests(unittest.TestCase): - def test_source_document_is_engineering_managed_without_owned_fields(self): - class Client: - def __init__(self): - self.updated = [] - - def list_rows(self, table_id): - return [ - { - "id": 7, - "Production Key": "REQ-1", - "Source Document": "A-26C-OLD", - "Status": "On Machine", - "Machinist": "Corey", - "QC Outcome": "Not Inspected", - "Disposition": "Make", - "Claimed Quantity": 2, - "Completed At": "2026-08-01T12:00:00Z", - } - ] - - def batch_create(self, table_id, rows): - raise AssertionError("The existing requirement should be updated") - - def batch_update(self, table_id, rows): - self.updated.extend(rows) - return rows - - client = Client() - desired = [ - { - "Production Key": "REQ-1", - "Source Document": "A-26C-0001", - } - ] - - created, updated, unchanged = MODULE.upsert_table( - client, - 1119642, - "Production Key", - desired, - MODULE.PRODUCTION_REQUIREMENT_MANAGED_FIELDS, - change_flag_field="Engineering Changed", - ) - - self.assertEqual((created, updated, unchanged), (0, 1, 0)) - self.assertEqual(client.updated[0]["Source Document"], "A-26C-0001") - self.assertTrue(client.updated[0]["Engineering Changed"]) - for field in ( - "Status", - "Machinist", - "QC Outcome", - "Disposition", - "Claimed Quantity", - "Completed At", - ): - self.assertNotIn(field, client.updated[0]) - - def test_finishing_upsert_preserves_manually_assigned_machinist(self): - class Client: - def __init__(self): - self.updated = [] - - def list_rows(self, table_id): - return [ - { - "id": 7, - "Production Key": "REQ-1", - "Production Requirement": [3], - "Powder Coat Color": "Red", - "Required Quantity": 1, - "Active": True, - "Last Synced At": "old", - "Machinist": "Corey", - } - ] - - def batch_create(self, table_id, rows): - self.fail("The existing finishing row should be updated") - - def batch_update(self, table_id, rows): - self.updated.extend(rows) - return rows - - client = Client() - desired = [ - { - "Production Key": "REQ-1", - "Production Requirement": [3], - "Powder Coat Color": "Black", - "Required Quantity": 4, - "Active": True, - "Last Synced At": "new", - } - ] - managed_fields = ( - "Production Requirement", - "Powder Coat Color", - "Required Quantity", - "Active", - "Last Synced At", - ) - - created, updated, unchanged = MODULE.upsert_table( - client, 6, "Production Key", desired, managed_fields - ) - - self.assertEqual((created, updated, unchanged), (0, 1, 0)) - self.assertEqual(len(client.updated), 1) - self.assertNotIn("Machinist", client.updated[0]) - - def test_root_revision_gate_matches_baserow_and_checks_discovery_membership(self): - released = MODULE.released_assembly_from_revision( - revision("B", VID_B, partNumber="A-ROOT-ONE") - ) - discovery_master = "https://cad.onshape.com/documents/master/w/main/e/assembly" - - class Client: - def list_rows(self, table_id): - self.table_id = table_id - return [ - { - "Assembly Number": "A-ROOT-ONE", - "Latest Released Revision": "B", - "Sync Schema Version": MODULE.SYNC_SCHEMA_VERSION, - "Discovery Master": discovery_master, - "Integration Status": "Discovered — Master Unreleased", - } - ] - - client = Client() - env = { - "BASEROW_API_URL": "https://api.baserow.test/api", - "BASEROW_TOKEN": "test", - "BASEROW_ASSEMBLIES_TABLE_ID": "4", - } - with patch.dict(os.environ, env), patch.object( - MODULE, "BaserowClient", return_value=client - ): - self.assertTrue( - MODULE.all_root_revisions_are_current( - [released], discovery_master - ) - ) - - self.assertEqual(client.table_id, 4) - - def test_root_revision_gate_detects_changed_revision_or_discovery_membership(self): - released = MODULE.released_assembly_from_revision( - revision("C", VID_B, partNumber="A-ROOT-ONE") - ) - discovery_master = "https://cad.onshape.com/documents/master/w/main/e/assembly" - - class Client: - def list_rows(self, table_id): - return [ - { - "Assembly Number": "A-ROOT-ONE", - "Latest Released Revision": "B", - "Discovery Master": discovery_master, - "Integration Status": "Discovered — Master Unreleased", - }, - { - "Assembly Number": "A-ROOT-TWO", - "Latest Released Revision": "D", - "Discovery Master": discovery_master, - "Integration Status": "Discovered — Master Unreleased", - }, - ] - - env = { - "BASEROW_API_URL": "https://api.baserow.test/api", - "BASEROW_TOKEN": "test", - "BASEROW_ASSEMBLIES_TABLE_ID": "4", - } - with patch.dict(os.environ, env), patch.object( - MODULE, "BaserowClient", return_value=Client() - ): - self.assertFalse(MODULE.all_root_revisions_are_current([released])) - - same_revision = replace(released, revision="B") - self.assertFalse( - MODULE.all_root_revisions_are_current( - [same_revision], discovery_master - ) - ) - - def test_batch_create_error_includes_response_and_machine_values(self): - response = RejectingResponse( - { - "error": "ERROR_REQUEST_BODY_VALIDATION", - "detail": { - "items": { - "1": {"Machine OP2": [{"error": "Invalid select option"}]} - } - }, - }, - status_code=400, - ) - - class Session: - def post(self, url, json, timeout): - return response - - client = object.__new__(MODULE.BaserowClient) - client.base_url = "https://api.baserow.test/api" - client.session = Session() - items = [ - { - "Production Key": "A-ROOT|A|A-ROOT|P-ONE|default", - "Machine OP1": "Haas CNC", - "Machine OP2": None, - }, - { - "Production Key": "A-ROOT|A|A-ROOT|P-TWO|default", - "Machine OP1": "Shop Sabre CNC", - "Machine OP2": "Haas CNC", - }, - ] - - with self.assertRaises(RuntimeError) as raised: - client.batch_create(1119642, items) - - message = str(raised.exception) - self.assertIn("Baserow batch create failed for table 1119642", message) - self.assertIn("ERROR_REQUEST_BODY_VALIDATION", message) - self.assertIn('"batch_index": 1', message) - self.assertIn("A-ROOT|A|A-ROOT|P-TWO|default", message) - self.assertIn('"Machine OP2": "Haas CNC"', message) - - class MultiRootSyncTests(unittest.TestCase): def test_unchanged_production_run_stops_before_fetching_root_bom(self): released = MODULE.released_assembly_from_revision( @@ -1748,11 +1292,11 @@ def test_unchanged_production_run_stops_before_fetching_root_bom(self): with patch.object( MODULE, "resolve_latest_released_assembly", return_value=released ), patch.object( - MODULE, "all_root_revisions_are_current", return_value=True + MODULE, "stale_root_revisions", return_value=(set(), False) ) as revision_gate, patch.object( MODULE, "fetch_bom", side_effect=AssertionError("BOM fetched") ), patch.object( - MODULE, "sync_to_baserow", side_effect=AssertionError("full sync started") + MODULE, "sync_to_supabase", side_effect=AssertionError("full sync started") ): result = MODULE.run_sync([target()], ["P-190B-26"]) @@ -1760,6 +1304,66 @@ def test_unchanged_production_run_stops_before_fetching_root_bom(self): self.assertEqual(result["roots_checked"], 1) revision_gate.assert_called_once() + def test_production_run_fetches_only_stale_root(self): + first_target = target() + second_target = MODULE.OnshapeTarget( + "https://cad.onshape.com", "1" * 24, "w", "2" * 24, "3" * 24 + ) + first_release = MODULE.released_assembly_from_revision( + revision("B", VID_B, partNumber="A-ROOT-ONE") + ) + second_release = MODULE.released_assembly_from_revision( + revision( + "D", + "4" * 24, + documentId=second_target.did, + elementId=second_target.eid, + partNumber="A-ROOT-TWO", + ) + ) + second_rows = [ + { + "item": "1", + "quantity": 1, + "partNumber": "P-190B-260102", + "name": "TWO", + "revision": "E", + "itemSource": source("https://example/two", 0), + } + ] + + with patch.object( + MODULE, + "resolve_latest_released_assembly", + side_effect=[first_release, second_release], + ), patch.object( + MODULE, + "stale_root_revisions", + return_value=({MODULE.normalized_part_number("A-ROOT-TWO")}, False), + ), patch.object( + MODULE, "fetch_bom", return_value=second_rows + ) as fetch_bom, patch.object( + MODULE, "hydrate_operation_properties", side_effect=lambda rows, *_: rows + ), patch.object( + MODULE, "source_document_names_for_rows", return_value=({}, []) + ), patch.object( + MODULE, "drawing_urls_for_parts", return_value=({}, []) + ), patch.object( + MODULE, "sync_to_supabase", return_value={"updated": 1} + ) as sync: + result = MODULE.run_sync( + [first_target, second_target], ["P-190B-26"] + ) + + self.assertEqual(result, {"updated": 1}) + fetch_bom.assert_called_once() + self.assertEqual(fetch_bom.call_args.args[0].did, second_release.document_id) + self.assertEqual(sync.call_args.kwargs["synced_roots"], {"A-ROOT-TWO"}) + self.assertEqual( + [row["Source Root"] for row in sync.call_args.args[1]], + ["A-ROOT-TWO"], + ) + def test_bad_list_root_is_logged_and_remaining_root_syncs(self): bad_target = target() good_target = MODULE.OnshapeTarget( @@ -1803,7 +1407,7 @@ def test_bad_list_root_is_logged_and_remaining_root_syncs(self): ) self.assertIn(MODULE.onshape_target_url(bad_target), warning) self.assertIn("RuntimeError: invalid latest-revision response", warning) - self.assertIn("existing Baserow requirements were left unchanged", warning) + self.assertIn("existing Supabase requirements were left unchanged", warning) self.assertTrue( any( call.args @@ -1816,24 +1420,24 @@ def test_bad_list_root_is_logged_and_remaining_root_syncs(self): result["source_revisions"][0]["part_number"], "A-ROOT-TWO" ) - def test_all_bad_list_roots_fail_before_baserow(self): + def test_all_bad_list_roots_fail_before_supabase(self): with patch.object( MODULE, "resolve_latest_released_assembly", side_effect=RuntimeError("invalid latest-revision response"), ), patch.object( MODULE, - "sync_to_baserow", - side_effect=AssertionError("Baserow called"), - ), self.assertRaisesRegex(RuntimeError, "Baserow was not changed"): + "sync_to_supabase", + side_effect=AssertionError("Supabase called"), + ), self.assertRaisesRegex(RuntimeError, "Supabase was not changed"): MODULE.run_sync([target()], ["P-190B-26"]) - def test_no_released_direct_children_fails_before_baserow(self): + def test_no_released_direct_children_fails_before_supabase(self): with patch.object(MODULE, "fetch_bom", return_value=[]), patch.object( MODULE, - "sync_to_baserow", - side_effect=AssertionError("Baserow called"), - ), self.assertRaisesRegex(RuntimeError, "Baserow was not changed"): + "sync_to_supabase", + side_effect=AssertionError("Supabase called"), + ), self.assertRaisesRegex(RuntimeError, "Supabase was not changed"): MODULE.run_sync( target(), ["P-190B-26"], discover_from_master=True ) @@ -2004,279 +1608,193 @@ def test_independent_roots_use_master_only_for_revision_comparison(self): "Current in Master", ) - def test_deactivation_is_limited_to_the_current_source_root(self): - table_ids = { - "sync": 1, - "parts": 2, - "requirements": 3, - "assemblies": 4, - "operations": 5, - "finishing": 6, - } - desired_part = { - "Part Number": "P-190B-260101", - "Name": "ONE", - "Description": "", - "Material": "", - "Manufacturing Method": "", - "Vendor": "", - "Revision": "C", - "OnShape Text": "RELEASED", - "Category": "", - "Onshape Drawing": "", - "Active": True, - } - class Client: - def __init__(self): - self.rows = { - table_ids["assemblies"]: [ - { - "id": 11, - "Assembly Number": "A-ROOT-TWO", - "Active": True, - "Discovery Master": "https://example/master", - "Integration Status": "Discovered — Master Unreleased", - }, - ], - table_ids["parts"]: [{"id": 20, **desired_part}], - table_ids["requirements"]: [ - { - "id": 30, - "Production Key": "old-one", - "Source Root": "A-ROOT-ONE", - "Assembly": [{"id": 10, "value": "A-ROOT-ONE"}], - "Machine OP1": None, - "Machine OP2": None, - "Machine OP3": None, - "Machine OP4": None, - "Finishing": None, - "Active in BOM": True, - }, - { - "id": 31, - "Production Key": "other-root", - "Source Root": "A-ROOT-TWO", - "Assembly": [{"id": 11, "value": "A-ROOT-TWO"}], - "Active in BOM": True, - }, - { - "id": 32, - "Production Key": "legacy-current-root", - "Assembly": [{"id": 10, "value": "A-ROOT-ONE"}], - "Active in BOM": True, - }, - { - "id": 33, - "Production Key": "legacy-other-root", - "Assembly": [{"id": 11, "value": "A-ROOT-TWO"}], - "Active in BOM": True, - }, - ], - table_ids["operations"]: [ - { - "id": 40, - "Operation": "stale-current-root", - "Production Requirement": [{"id": 30}], - "Operation Number": "OP4", - "Machine": "Haas CNC", - "Status": "In Progress", - "Active in Routing": True, - }, - { - "id": 41, - "Operation": "stale-other-root", - "Production Requirement": [{"id": 31}], - "Operation Number": "OP4", - "Machine": "Haas CNC", - "Status": "In Progress", - "Active in Routing": True, - }, - ], - table_ids["finishing"]: [ - { - "id": 60, - "Production Key": "stale-finishing", - "Production Requirement": [{"id": 30}], - "Active": True, - }, - { - "id": 61, - "Production Key": "other-root-finishing", - "Production Requirement": [{"id": 31}], - "Active": True, - }, - ], - } - self.updates = [] - self.next_id = 100 - - def create_one(self, table_id, fields): - return {"id": 1, **fields} - - def update_one(self, table_id, row_id, fields): - return {"id": row_id, **fields} - - def list_rows(self, table_id): - return list(self.rows[table_id]) - - def batch_create(self, table_id, rows): - created = [] - for row in rows: - self.next_id += 1 - item = {"id": self.next_id, **row} - self.rows[table_id].append(item) - created.append(item) - return created - - def batch_update(self, table_id, rows): - self.updates.append((table_id, list(rows))) - return rows - - client = Client() - requirement = { - "Production Key": ( - "A-ROOT-ONE|B|A-ROOT-ONE|P-190B-260101|default" - ), - "part_number": "P-190B-260101", - "assembly_number": "A-ROOT-ONE", - "Source Root": "A-ROOT-ONE", - "Source Assembly Revision": "B", - "Required Part Revision": "C", - "Configuration": "default", - "Required Quantity": 1, - "BOM Positions": "1", - "Onshape Source": "https://example/one", - "Source Document": "A-26C-0001", - "Machine OP1": "Haas CNC", - "Machine OP2": "Tapping", - "Machine OP3": None, - "Machine OP4": None, - "Finishing": "Red", - "Active in BOM": True, - } - env = { - "BASEROW_API_URL": "https://api.baserow.test/api", - "BASEROW_TOKEN": "test", - "BASEROW_SYNC_RUNS_TABLE_ID": "1", - "BASEROW_PARTS_TABLE_ID": "2", - "BASEROW_REQUIREMENTS_TABLE_ID": "3", - "BASEROW_ASSEMBLIES_TABLE_ID": "4", - "BASEROW_OPERATIONS_TABLE_ID": "5", - "BASEROW_FINISHING_TABLE_ID": "6", - } - operation_key = requirement["Production Key"] + "|OP1" - - with patch.dict(os.environ, env), patch.object( - MODULE, "BaserowClient", return_value=client - ): - MODULE.sync_to_baserow( - [desired_part], - [requirement], - [], - source_rows=1, - exports_by_part={}, - sync_cad_files=False, - operations=[ - { - "Operation": operation_key, - "production_key": requirement["Production Key"], - "Operation Number": "OP1", - "Machine": "Haas CNC", - "Active in Routing": True, - } - ], - synced_roots={"A-ROOT-ONE"}, - discovery_master="https://example/master", - ) - - requirement_updates = [ - row - for table_id, rows in client.updates - if table_id == table_ids["requirements"] - for row in rows - ] - deactivated_ids = { - row["id"] - for row in requirement_updates - if row.get("Active in BOM") is False - } - self.assertEqual(deactivated_ids, {30, 32}) - operation_updates = [ - row - for table_id, rows in client.updates - if table_id == table_ids["operations"] - for row in rows - ] - self.assertIn({"id": 40, "Active in Routing": False}, operation_updates) - self.assertNotIn({"id": 41, "Active in Routing": False}, operation_updates) - created_operation = next( - row - for row in client.rows[table_ids["operations"]] - if row["Operation"] == operation_key - ) - self.assertEqual(created_operation["Operation Number"], "OP1") - self.assertEqual(created_operation["Machine"], "Haas CNC") - self.assertEqual(created_operation["Status"], "Ready") - assembly_updates = [ - row - for table_id, rows in client.updates - if table_id == table_ids["assemblies"] - for row in rows - ] - self.assertIn( - { - "id": 11, - "Integration Status": "Missing from Main — Review", - }, - assembly_updates, - ) - root_row = next( - row - for row in client.rows[table_ids["assemblies"]] - if row["Assembly Number"] == "A-ROOT-ONE" - ) - created_requirement = next( - row - for row in client.rows[table_ids["requirements"]] - if row["Production Key"] == requirement["Production Key"] - ) - self.assertTrue(root_row["Active"]) - self.assertEqual(created_requirement["Assembly"], [root_row["id"]]) - self.assertEqual(created_requirement["Machine OP1"], "Haas CNC") - self.assertEqual(created_requirement["Machine OP2"], "Tapping") - self.assertIsNone(created_requirement["Machine OP3"]) - self.assertIsNone(created_requirement["Machine OP4"]) - self.assertEqual(created_requirement["Finishing"], "Red") - self.assertNotIn( - "Source Document", - created_requirement, - "Fields absent from the existing Baserow schema must stay filtered", - ) - finishing_row = next( - row - for row in client.rows[table_ids["finishing"]] - if row["Production Key"] == requirement["Production Key"] - ) - self.assertEqual( - finishing_row["Production Requirement"], [created_requirement["id"]] - ) - self.assertEqual(finishing_row["Powder Coat Color"], "Red") - self.assertEqual(finishing_row["Required Quantity"], 1) - self.assertTrue(finishing_row["Active"]) - finishing_updates = [ - row - for table_id, rows in client.updates - if table_id == table_ids["finishing"] - for row in rows - ] - self.assertTrue( - any( - row.get("id") == 60 and row.get("Active") is False - for row in finishing_updates - ) - ) - self.assertFalse(any(row.get("id") == 61 for row in finishing_updates)) +class SupabaseSyncTests(unittest.TestCase): + def test_secret_key_auth_and_rpc_use_public_endpoint(self): + session = Mock(headers={}) + session.post.return_value = FakeResponse([]) + with patch.object(MODULE.requests, "Session", return_value=session): + client = MODULE.SupabaseClient("https://project.supabase.test", "sb_secret_fixture") + self.assertEqual(session.headers, {"apikey": "sb_secret_fixture"}) + client.root_state() + self.assertEqual(session.post.call_args.args[0], + "https://project.supabase.test/rest/v1/rpc/manufacturing_engineering_sync_state") + self.assertNotIn("Accept-Profile", session.headers) + session.headers = {} + with patch.object(MODULE.requests, "Session", return_value=session): + MODULE.SupabaseClient("https://project.supabase.test", "fixture-jwt") + self.assertEqual(session.headers["Authorization"], "Bearer fixture-jwt") + + def test_private_storage_uses_content_address_and_verifies_duplicate_bytes(self): + session = Mock(headers={}) + session.post.return_value = FakeResponse({"error": "Duplicate"}, 409) + session.get.return_value = Mock(content=b"STEP") + with patch.object(MODULE.requests, "Session", return_value=session): + client = MODULE.SupabaseClient("https://project.supabase.test", "sb_secret_fixture") + result = client.upload_file("plate.step", b"STEP", "application/step") + digest = MODULE.hashlib.sha256(b"STEP").hexdigest() + self.assertEqual(result["storage_path"], f"sha256/{digest[:2]}/{digest}.step") + self.assertEqual(session.post.call_args.kwargs["headers"]["x-upsert"], "false") + self.assertNotIn("/public/", session.get.call_args.args[0]) + session.get.return_value.content = b"wrong" + with self.assertRaisesRegex(RuntimeError, "verification"): + client.upload_file("plate.step", b"STEP", "application/step") + + def test_storage_permission_error_is_not_a_duplicate(self): + client = object.__new__(MODULE.SupabaseClient) + client.base_url = "https://project.supabase.test" + client.session = Mock() + client.session.post.return_value = RejectingResponse({"error": "AccessDenied"}, 403) + with self.assertRaises(RuntimeError): + client.upload_file("plate.pdf", b"PDF", "application/pdf") + client.session.get.assert_not_called() + + def test_payload_is_one_transaction_with_business_keys_and_no_shop_fields(self): + client = Mock() + client.rpc.return_value = {"status": "success"} + requirements = [{"Production Key": "ROOT|A|ROOT|P|default", "part_number": "P", + "assembly_number": "ROOT", "Source Root": "ROOT", "Finishing": "Red", + "Required Quantity": 4, "Status": "DO NOT SEND", "Machinist": "DO NOT SEND", + "QC Outcome": "DO NOT SEND", "location_id": 9}] + operations = [{"Operation": "ROOT|A|ROOT|P|default|OP1", "production_key": requirements[0]["Production Key"], + "Operation Number": "OP1", "Machine": "Haas CNC", "Active in Routing": True, + "Status": "Ready", "Claimed Quantity": 6, "Completed Quantity": 2}] + with patch.object(MODULE.SupabaseClient, "from_env", return_value=client): + MODULE.sync_to_supabase([{"Part Number": "P", "Name": "Plate"}], requirements, [], 1, + {}, False, operations=operations, synced_roots={"ROOT"}, run_id="fixture") + client.rpc.assert_called_once() + payload = client.rpc.call_args.kwargs["p_payload"] + self.assertEqual(payload["requirements"][0]["part_number"], "P") + self.assertNotIn("part_id", payload["requirements"][0]) + self.assertEqual(payload["operations"][0]["production_key"], requirements[0]["Production Key"]) + self.assertEqual(payload["finishing"][0]["required_quantity"], 4) + self.assertNotIn("DO NOT SEND", json.dumps(payload)) + self.assertNotIn("status", payload["operations"][0]) + self.assertNotIn("claimed_quantity", payload["operations"][0]) + client.attachment_state.assert_not_called() + + def test_database_failure_is_reported_and_not_retried_as_table_writes(self): + client = Mock() + client.rpc.return_value = {"status": "failed", "error": "invalid relation"} + with patch.object(MODULE.SupabaseClient, "from_env", return_value=client): + with self.assertRaisesRegex(RuntimeError, "rolled back"): + MODULE.sync_to_supabase([], [], [], 0, {}, False, run_id="fixture") + client.rpc.assert_called_once() + client.finish_run.assert_called_once() + + def test_main_records_onshape_failure_and_skip_without_credentials_in_dry_run(self): + client = Mock() + client.begin_run.return_value = "fixture" + env = {"ONSHAPE_DOC_URL": MODULE.onshape_target_url(target())} + with patch.dict(os.environ, env, clear=True), patch.object( + MODULE.SupabaseClient, "from_env", return_value=client + ), patch.object(MODULE, "run_sync", side_effect=RuntimeError("Onshape unavailable")): + with self.assertRaisesRegex(RuntimeError, "Onshape unavailable"): + MODULE.main([]) + self.assertEqual(client.finish_run.call_args.args[1], "failed") + client.reset_mock() + with patch.dict(os.environ, env, clear=True), patch.object( + MODULE.SupabaseClient, "from_env", return_value=client + ), patch.object(MODULE, "run_sync", return_value={"skipped": True, "warnings": ["root unresolved"]}): + MODULE.main([]) + self.assertEqual(client.finish_run.call_args.args[1], "partial") + with patch.dict(os.environ, env, clear=True), patch.object( + MODULE.SupabaseClient, "from_env", side_effect=AssertionError("destination accessed") + ), patch.object(MODULE, "run_sync", return_value={"dry_run": True}): + self.assertEqual(MODULE.main(["--dry-run"]), 0) + + def test_revision_gate_requires_cad_completion_only_when_requested(self): + released = MODULE.released_assembly_from_revision(revision("B", VID_B)) + client = Mock() + client.root_state.return_value = [{"Assembly Number": released.part_number, + "Latest Released Revision": "B", "Sync Schema Version": MODULE.SYNC_SCHEMA_VERSION, + "CAD Synced": False}] + with patch.object(MODULE.SupabaseClient, "from_env", return_value=client): + self.assertEqual(MODULE.stale_root_revisions([released]), (set(), False)) + self.assertTrue(MODULE.stale_root_revisions([released], sync_cad_files=True)[0]) + client.root_state.return_value[0]["CAD Synced"] = True + self.assertEqual(MODULE.stale_root_revisions([released], sync_cad_files=True), (set(), False)) + client.root_state.return_value[0]["Sync Schema Version"] = "old" + self.assertTrue(MODULE.stale_root_revisions([released])[0]) + + def test_membership_change_does_not_rescan_unchanged_bom_or_drawings(self): + released = MODULE.released_assembly_from_revision(revision("B", VID_B)) + reference = MODULE.OnshapeDocumentReference(target().base_url,DID,"w",WID) + with patch.object(MODULE, "fetch_bom", return_value=[]) as bom, patch.object( + MODULE, "discover_released_manufacturing_roots", return_value=([(reference,released)], []) + ), patch.object(MODULE, "stale_root_revisions", return_value=(set(), True)), patch.object( + MODULE, "drawing_urls_for_parts", side_effect=AssertionError("drawings rescanned") + ), patch.object(MODULE, "sync_to_supabase", return_value={"status": "success"}) as sync: + MODULE.run_sync(target(), [], discover_from_master=True) + bom.assert_called_once() # Main discovery only. + self.assertEqual(sync.call_args.kwargs["synced_roots"], set()) + self.assertEqual(sync.call_args.kwargs["discovered_roots"], {released.part_number}) + + def test_no_revision_never_reaches_drawings(self): + with patch.object(MODULE, "resolve_latest_released_assembly", side_effect=RuntimeError("No released revision")), patch.object( + MODULE, "fetch_bom", side_effect=AssertionError("BOM fetched") + ), patch.object(MODULE, "drawing_urls_for_parts", side_effect=AssertionError("drawings scanned")): + with self.assertRaisesRegex(RuntimeError, "No configured"): + MODULE.run_sync(target(), [], dry_run=True) + + def test_failed_root_bom_is_excluded_from_deactivation_scope(self): + one = MODULE.released_assembly_from_revision(revision("B", VID_B, partNumber="A-ONE")) + two = replace(one, part_number="A-TWO") + with patch.object(MODULE, "resolve_latest_released_assembly", side_effect=[one,two]), patch.object( + MODULE, "stale_root_revisions", return_value=({"a-one","a-two"},False) + ), patch.object(MODULE, "fetch_bom", side_effect=[RuntimeError("timeout"),[]]), patch.object( + MODULE, "drawing_urls_for_parts", return_value=({},[]) + ), patch.object(MODULE, "sync_to_supabase", return_value={"status":"partial"}) as sync: + MODULE.run_sync([target(),replace(target(),did="1"*24)], []) + self.assertEqual(sync.call_args.kwargs["synced_roots"], {"A-TWO"}) + self.assertIn("timeout", sync.call_args.args[2][0]) + + def test_dry_run_discovery_never_generates_bom_or_translates(self): + released = MODULE.released_assembly_from_revision(revision("B",VID_B)) + reference = MODULE.OnshapeDocumentReference(target().base_url,DID,"w",WID) + with patch.dict(os.environ, {}, clear=True), patch.object(MODULE, "fetch_bom", return_value=[]) as bom, patch.object( + MODULE, "discover_released_manufacturing_roots", return_value=([(reference,released)],[]) + ), patch.object(MODULE, "drawing_urls_for_parts", return_value=({},[])), patch.object( + MODULE, "start_file_translation", side_effect=AssertionError("mutation") + ), patch.object(MODULE.SupabaseClient, "from_env", side_effect=AssertionError("credentials")): + MODULE.run_sync(target(), [], dry_run=True, sync_cad_files=True, discover_from_master=True) + self.assertFalse(bom.call_args_list[0].kwargs["generate_if_absent"]) + + def test_cached_export_group_skips_onshape_translation(self): + export = FileExportTests().sample_export() + state = [{"part_number": export.part_number, "kind": "step", "file_count": 1, + "export_key": MODULE.aggregate_export_key([export])}] + with patch.object(MODULE, "start_file_translation", side_effect=AssertionError("translation")): + groups,cached = MODULE.attach_exported_files(object(), [{"Part Number":export.part_number}], + state, {export.part_number:[export]}, []) + self.assertEqual((groups,cached), ([],1)) + + def test_incomplete_export_group_never_updates_catalog_or_marker(self): + export = FileExportTests().sample_export() + second = replace(export, source_key="second") + client=Mock() + client.upload_file.side_effect=[{"original_name":"first.step"},RuntimeError("upload failed")] + warnings=[] + with patch.object(MODULE,"start_file_translation",return_value={}), patch.object( + MODULE,"wait_for_translation",return_value={} + ), patch.object(MODULE,"download_translation",return_value=b"STEP"): + groups,cached=MODULE.attach_exported_files(client,[{"Part Number":export.part_number}],[], + {export.part_number:[export,second]},warnings) + self.assertEqual(groups,[]) + self.assertIn("upload failed",warnings[0]) + + def test_completed_export_retains_configured_filename(self): + export=FileExportTests().sample_export() + client=Mock() + client.upload_file.return_value={"original_name":"Configured Shop Export.step"} + with patch.object(MODULE,"start_file_translation",return_value={}), patch.object( + MODULE,"wait_for_translation",return_value={"exportRuleFileName":"Configured Shop Export"} + ), patch.object(MODULE,"download_translation",return_value=b"STEP"): + groups,cached=MODULE.attach_exported_files(client,[{"Part Number":export.part_number}],[], + {export.part_number:[export]},[]) + self.assertEqual(client.upload_file.call_args.args[0],"Configured Shop Export.step") + self.assertEqual(groups[0]["export_key"],MODULE.aggregate_export_key([export])) + self.assertEqual(groups[0]["files"][0]["source_metadata"]["request"],export.request_body) if __name__ == "__main__": diff --git a/supabase/.temp/cli-latest b/supabase/.temp/cli-latest new file mode 100644 index 0000000..a0df631 --- /dev/null +++ b/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.116.0 \ No newline at end of file diff --git a/supabase/production/20260906_onshape_engineering_sync.sql b/supabase/production/20260906_onshape_engineering_sync.sql new file mode 100644 index 0000000..7dc4e11 --- /dev/null +++ b/supabase/production/20260906_onshape_engineering_sync.sql @@ -0,0 +1,408 @@ +-- Engineering sync API for 190-Manufacturing experimental b915b1439ae4fed616f24935de552734ebea525b. +-- Apply manually AFTER normalized_manufacturing and manufacturing_attachments. +-- This repository does not apply this migration or change PostgREST exposure. +begin; + +alter table manufacturing.assemblies + add column latest_released_revision text, + add column master_baseline_revision text, + add column integration_status text, + add column discovery_master text, + add column onshape_url text, + add column last_synced_at timestamptz, + add column cad_synced boolean not null default false; +-- Imported field identifiers remain intact. New Onshape attachments have no legacy field ID. +alter table manufacturing.attachments alter column source_field_id drop not null; +-- Retain the contract's nonunique operations_key_idx and existing shop records. + +create table manufacturing.engineering_sync_runs ( + id uuid primary key, + started_at timestamptz not null default clock_timestamp(), + finished_at timestamptz, + status text not null default 'running' check (status in ('running','success','partial','failed')), + github_run_url text not null, + summary jsonb not null default '{}' +); +alter table manufacturing.engineering_sync_runs enable row level security; +revoke all on manufacturing.engineering_sync_runs from public, anon, authenticated, service_role; + +create function public.manufacturing_begin_engineering_sync(p_run_id uuid, p_run_url text) +returns uuid language plpgsql security definer set search_path = '' as $$ +begin + insert into manufacturing.engineering_sync_runs(id, github_run_url) + values(p_run_id, coalesce(p_run_url, '')) on conflict(id) do nothing; + return p_run_id; +end; +$$; + +create function public.manufacturing_finish_engineering_sync(p_run_id uuid, p_status text, p_summary jsonb) +returns void language plpgsql security definer set search_path = '' as $$ +begin + if p_status is null or p_status not in ('success','partial','failed') or p_summary is null then + raise exception 'Invalid engineering sync result'; + end if; + -- An uncertain HTTP response must never turn a committed success into a failure. + update manufacturing.engineering_sync_runs + set status=p_status, summary=p_summary, finished_at=clock_timestamp() + where id=p_run_id and status='running'; +end; +$$; + +create function public.manufacturing_engineering_sync_state() +returns jsonb language sql stable security definer set search_path = '' as $$ + select coalesce(jsonb_agg(jsonb_build_object( + 'Assembly Number', assembly_number, 'Latest Released Revision', latest_released_revision, + 'Sync Schema Version', sync_schema_version, 'Discovery Master', discovery_master, + 'Integration Status', integration_status, 'CAD Synced', cad_synced + )), '[]'::jsonb) from manufacturing.assemblies; +$$; + +create function public.manufacturing_engineering_file_state(p_part_numbers text[]) +returns jsonb language sql stable security definer set search_path = '' as $$ + select coalesce(jsonb_agg(to_jsonb(s)), '[]'::jsonb) from ( + select p.part_number, a.kind, + case a.kind when 'drawing-pdf' then p.drawing_export_key else p.step_export_key end as export_key, + count(*) as file_count + from manufacturing.parts p join manufacturing.attachments a on a.part_id=p.id + where p.part_number=any(p_part_numbers) group by p.id, a.kind + ) s; +$$; + +create function public.manufacturing_apply_engineering_sync(p_run_id uuid, p_payload jsonb) +returns jsonb language plpgsql security definer set search_path = '' as $$ +declare + run manufacturing.engineering_sync_runs; + entity text; row_key text; allowed text[]; v jsonb; f jsonb; g jsonb; + roots text[]; part_id_value bigint; requirement_id_value bigint; assembly_id_value bigint; + position_value integer; result_summary jsonb; error_message text; + requirements_deactivated integer; operations_deactivated integer; finishing_deactivated integer; + operation_matches integer; duplicate_operations_preserved integer := 0; +begin + -- Serialize engineering commits, while shop transactions retain their ordinary locks. + perform pg_catalog.pg_advisory_xact_lock(190, 20260905); + select * into strict run from manufacturing.engineering_sync_runs where id=p_run_id for update; + if run.status <> 'running' then return run.summary; end if; + -- A subtransaction rolls back EVERY BOM/catalog change on error, retaining the audit result. + begin + if p_payload is null or jsonb_typeof(p_payload) <> 'object' then + raise exception 'Invalid engineering payload'; + end if; + foreach entity in array array['attachments','warnings','discovered_roots'] loop + if jsonb_typeof(p_payload->entity) is distinct from 'array' then + raise exception 'Missing engineering array: %', entity; + end if; + end loop; + if exists(select 1 from manufacturing.engineering_sync_runs + where id<>p_run_id and summary->>'committed'='true' and finished_at>run.started_at) then + raise exception 'Another engineering run committed after this run started; retry from fresh state'; + end if; + if jsonb_typeof(p_payload->'synced_roots') is distinct from 'array' then + raise exception 'Successful root scope is required'; + end if; + select coalesce(array_agg(value),array[]::text[]) into roots from jsonb_array_elements_text(p_payload->'synced_roots'); + if cardinality(roots)=0 and ( + coalesce(p_payload->>'discovery_master','')='' or exists( + select 1 from unnest(array['assemblies','parts','requirements','operations','finishing','attachments']) e + where jsonb_array_length(p_payload->e)>0)) then + raise exception 'Empty scope is only allowed for discovery membership bookkeeping'; + end if; + if exists(select 1 from unnest(roots) r where r is null or btrim(r)='') then + raise exception 'Empty root scope'; + end if; + -- Explicit allowlists reject shop fields and unknown columns, even for service_role. + foreach entity in array array['assemblies','parts','requirements','operations','finishing'] loop + case entity + when 'assemblies' then allowed:=array['assembly_number','subsystem_name','active','sync_schema_version','latest_released_revision','master_baseline_revision','integration_status','discovery_master','onshape_url','last_synced_at']; row_key:='assembly_number'; + when 'parts' then allowed:=array['part_number','name','description','material','manufacturing_method','vendor','revision','onshape_url','category','drawing_url','active']; row_key:='part_number'; + when 'requirements' then allowed:=array['production_key','part_number','assembly_number','configuration','required_quantity','bom_positions','onshape_url','source_document','source_root','source_assembly_revision','required_part_revision','machine_op1','machine_op2','machine_op3','machine_op4','finishing','active_in_bom']; row_key:='production_key'; + when 'operations' then allowed:=array['operation_key','production_key','operation_number','machine','active_in_routing','work_type']; row_key:='operation_key'; + when 'finishing' then allowed:=array['production_key','color','required_quantity','active']; row_key:='production_key'; + end case; + if jsonb_typeof(p_payload->entity) is distinct from 'array' then + raise exception 'Missing engineering array: %', entity; + end if; + for v in select value from jsonb_array_elements(p_payload->entity) loop + if jsonb_typeof(v)<>'object' or coalesce(btrim(v->>row_key),'')='' then + raise exception 'Missing business key for %', entity; + end if; + if exists(select 1 from jsonb_object_keys(v) k where not(k=any(allowed))) then + raise exception 'Unknown or shop-owned field in %', entity; + end if; + end loop; + if exists(select 1 from jsonb_array_elements(p_payload->entity) r + group by r->>row_key having count(*)>1) then + raise exception 'Duplicate business key in %', entity; + end if; + end loop; + if exists(select 1 from unnest(roots) root where not exists( + select 1 from jsonb_array_elements(p_payload->'assemblies') a + where a->>'assembly_number'=root and coalesce(a->>'latest_released_revision','')<>'')) then + raise exception 'Every successful root must have a released assembly record'; + end if; + + for v in select value from jsonb_array_elements(p_payload->'assemblies') loop + v:=v || jsonb_build_object('last_synced_at',clock_timestamp()); + if v->>'assembly_number'=any(roots) then v:=v || jsonb_build_object('cad_synced',coalesce((p_payload->>'cad_synced')::boolean,false)); end if; + if not(v ? 'cad_synced') then v:=v || jsonb_build_object('cad_synced',false); end if; + insert into manufacturing.assemblies as existing(assembly_number, subsystem_name, active, sync_schema_version, latest_released_revision, master_baseline_revision, integration_status, discovery_master, onshape_url, last_synced_at, cad_synced) + select assembly_number, subsystem_name, active, sync_schema_version, latest_released_revision, master_baseline_revision, integration_status, discovery_master, onshape_url, last_synced_at, cad_synced from jsonb_populate_record(null::manufacturing.assemblies,v) + on conflict (assembly_number) where assembly_number is not null and assembly_number<>'' do update set + subsystem_name=case when v ? 'subsystem_name' then excluded.subsystem_name else existing.subsystem_name end, + active=case when v ? 'active' then excluded.active else existing.active end, + sync_schema_version=case when v ? 'sync_schema_version' then excluded.sync_schema_version else existing.sync_schema_version end, + latest_released_revision=case when v ? 'latest_released_revision' then excluded.latest_released_revision else existing.latest_released_revision end, + master_baseline_revision=case when v ? 'master_baseline_revision' then excluded.master_baseline_revision else existing.master_baseline_revision end, + integration_status=case when v ? 'integration_status' then excluded.integration_status else existing.integration_status end, + discovery_master=case when v ? 'discovery_master' then excluded.discovery_master else existing.discovery_master end, + onshape_url=case when v ? 'onshape_url' then excluded.onshape_url else existing.onshape_url end, + last_synced_at=case when v ? 'last_synced_at' then excluded.last_synced_at else existing.last_synced_at end, + cad_synced=case when v->>'assembly_number'=any(roots) then excluded.cad_synced else existing.cad_synced end, + updated_at=clock_timestamp(); + end loop; + + for v in select value from jsonb_array_elements(p_payload->'parts') loop + v:=v || jsonb_build_object('last_synced_at',clock_timestamp()); + insert into manufacturing.parts as existing(part_number, name, description, material, manufacturing_method, vendor, revision, onshape_url, category, drawing_url, active, last_synced_at) + select part_number, name, description, material, manufacturing_method, vendor, revision, onshape_url, category, drawing_url, active, last_synced_at from jsonb_populate_record(null::manufacturing.parts,v) + on conflict (part_number) where part_number is not null and part_number<>'' do update set + name=case when v ? 'name' then excluded.name else existing.name end, + description=case when v ? 'description' then excluded.description else existing.description end, + material=case when v ? 'material' then excluded.material else existing.material end, + manufacturing_method=case when v ? 'manufacturing_method' then excluded.manufacturing_method else existing.manufacturing_method end, + vendor=case when v ? 'vendor' then excluded.vendor else existing.vendor end, + revision=case when v ? 'revision' then excluded.revision else existing.revision end, + onshape_url=case when v ? 'onshape_url' then excluded.onshape_url else existing.onshape_url end, + category=case when v ? 'category' then excluded.category else existing.category end, + drawing_url=case when v ? 'drawing_url' then excluded.drawing_url else existing.drawing_url end, + active=case when v ? 'active' then excluded.active else existing.active end, + last_synced_at=case when v ? 'last_synced_at' then excluded.last_synced_at else existing.last_synced_at end, + updated_at=clock_timestamp(); + end loop; + + for v in select value from jsonb_array_elements(p_payload->'requirements') loop + if not(coalesce(v->>'source_root','')=any(roots)) then + raise exception 'Requirement outside successful root scope'; + end if; + if not exists(select 1 from jsonb_array_elements(p_payload->'assemblies') a + where a->>'assembly_number'=v->>'source_root' + and a->>'latest_released_revision'=v->>'source_assembly_revision') then + raise exception 'Requirement revision differs from successful root'; + end if; + if not exists(select 1 from jsonb_array_elements(p_payload->'parts') p + where p->>'part_number'=v->>'part_number') then + raise exception 'Requirement part is absent from payload'; + end if; + select id into strict part_id_value from manufacturing.parts where part_number=v->>'part_number'; + assembly_id_value:=null; + if coalesce(v->>'assembly_number','')<>'' then + select id into strict assembly_id_value from manufacturing.assemblies where assembly_number=v->>'assembly_number'; + end if; + if exists(select 1 from manufacturing.requirements r where r.production_key=v->>'production_key' + and coalesce(r.source_root,'')<>'' and r.source_root<>v->>'source_root') then + raise exception 'Cannot move an existing requirement between roots'; + end if; + if (v->>'required_quantity')::numeric is null or (v->>'required_quantity')::numeric < 0 + or (v->>'required_quantity')::numeric::text in ('NaN','Infinity','-Infinity') then + raise exception 'Invalid required engineering quantity'; + end if; + v:=(v-'part_number'-'assembly_number') || jsonb_build_object('part_id',part_id_value, + 'assembly_id',assembly_id_value, 'engineering_changed',false); + v:=v || jsonb_build_object('last_synced_at',clock_timestamp()); + insert into manufacturing.requirements as existing(production_key, configuration, required_quantity, bom_positions, onshape_url, source_document, source_root, source_assembly_revision, required_part_revision, machine_op1, machine_op2, machine_op3, machine_op4, finishing, active_in_bom, part_id, assembly_id, engineering_changed, last_synced_at) + select production_key, configuration, required_quantity, bom_positions, onshape_url, source_document, source_root, source_assembly_revision, required_part_revision, machine_op1, machine_op2, machine_op3, machine_op4, finishing, active_in_bom, part_id, assembly_id, engineering_changed, last_synced_at from jsonb_populate_record(null::manufacturing.requirements,v) + on conflict (production_key) where production_key is not null and production_key<>'' do update set + configuration=case when v ? 'configuration' then excluded.configuration else existing.configuration end, + required_quantity=case when v ? 'required_quantity' then excluded.required_quantity else existing.required_quantity end, + bom_positions=case when v ? 'bom_positions' then excluded.bom_positions else existing.bom_positions end, + onshape_url=case when v ? 'onshape_url' then excluded.onshape_url else existing.onshape_url end, + source_document=case when v ? 'source_document' then excluded.source_document else existing.source_document end, + source_root=case when v ? 'source_root' then excluded.source_root else existing.source_root end, + source_assembly_revision=case when v ? 'source_assembly_revision' then excluded.source_assembly_revision else existing.source_assembly_revision end, + required_part_revision=case when v ? 'required_part_revision' then excluded.required_part_revision else existing.required_part_revision end, + machine_op1=case when v ? 'machine_op1' then excluded.machine_op1 else existing.machine_op1 end, + machine_op2=case when v ? 'machine_op2' then excluded.machine_op2 else existing.machine_op2 end, + machine_op3=case when v ? 'machine_op3' then excluded.machine_op3 else existing.machine_op3 end, + machine_op4=case when v ? 'machine_op4' then excluded.machine_op4 else existing.machine_op4 end, + finishing=case when v ? 'finishing' then excluded.finishing else existing.finishing end, + active_in_bom=case when v ? 'active_in_bom' then excluded.active_in_bom else existing.active_in_bom end, + part_id=case when v ? 'part_id' then excluded.part_id else existing.part_id end, + assembly_id=case when v ? 'assembly_id' then excluded.assembly_id else existing.assembly_id end, + engineering_changed=coalesce(existing.engineering_changed,false) or + row(existing.configuration,existing.required_quantity,existing.bom_positions,existing.onshape_url,existing.source_document,existing.source_root,existing.source_assembly_revision,existing.required_part_revision,existing.machine_op1,existing.machine_op2,existing.machine_op3,existing.machine_op4,existing.finishing,existing.active_in_bom,existing.part_id,existing.assembly_id) + is distinct from row(excluded.configuration,excluded.required_quantity,excluded.bom_positions,excluded.onshape_url,excluded.source_document,excluded.source_root,excluded.source_assembly_revision,excluded.required_part_revision,excluded.machine_op1,excluded.machine_op2,excluded.machine_op3,excluded.machine_op4,excluded.finishing,excluded.active_in_bom,excluded.part_id,excluded.assembly_id), + last_synced_at=case when v ? 'last_synced_at' then excluded.last_synced_at else existing.last_synced_at end, + updated_at=clock_timestamp(); + end loop; + + for v in select value from jsonb_array_elements(p_payload->'operations') loop + if not exists(select 1 from jsonb_array_elements(p_payload->'requirements') r + where r->>'production_key'=v->>'production_key') then + raise exception 'Work row requirement is absent from successful scope'; + end if; + select id into strict requirement_id_value from manufacturing.requirements + where production_key=v->>'production_key'; + v:=v || jsonb_build_object('requirement_id',requirement_id_value); + if v->>'work_type' is distinct from 'Manufacturing' + or v->>'operation_number' not in ('OP1','OP2','OP3','OP4') + or v->>'operation_key' is distinct from (v->>'production_key') || '|' || (v->>'operation_number') then + raise exception 'Invalid released operation identity'; + end if; + -- Lock all matches before validating identity. Never pick one duplicate + -- arbitrarily, combine quantities, or move another requirement's shop work. + perform o.id from manufacturing.operations o where o.operation_key=v->>'operation_key' for update; + if exists(select 1 from manufacturing.operations o where o.operation_key=v->>'operation_key' + and (o.requirement_id is distinct from requirement_id_value + or coalesce(o.work_type,'Manufacturing')<>'Manufacturing')) then + raise exception 'Existing operation key has incompatible requirement or work type'; + end if; + update manufacturing.operations o set + operation_number=v->>'operation_number', + machine=case when v ? 'machine' then v->>'machine' else o.machine end, + active_in_routing=case when v ? 'active_in_routing' then (v->>'active_in_routing')::boolean else o.active_in_routing end, + work_type='Manufacturing',updated_at=clock_timestamp() + where o.operation_key=v->>'operation_key'; + get diagnostics operation_matches = row_count; + if operation_matches=0 then + insert into manufacturing.operations(operation_key,operation_number,machine,active_in_routing,work_type,requirement_id) + select operation_key,operation_number,machine,active_in_routing,work_type,requirement_id + from jsonb_populate_record(null::manufacturing.operations,v); + elsif operation_matches>1 then + duplicate_operations_preserved:=duplicate_operations_preserved+operation_matches-1; + end if; + end loop; + + for v in select value from jsonb_array_elements(p_payload->'finishing') loop + if not exists(select 1 from jsonb_array_elements(p_payload->'requirements') r + where r->>'production_key'=v->>'production_key') then + raise exception 'Work row requirement is absent from successful scope'; + end if; + select id into strict requirement_id_value from manufacturing.requirements + where production_key=v->>'production_key'; + v:=v || jsonb_build_object('requirement_id',requirement_id_value); + if v->>'color' not in ('Red','Black') or not exists( + select 1 from manufacturing.requirements r where r.id=requirement_id_value + and r.finishing=v->>'color' and r.required_quantity=(v->>'required_quantity')::numeric) then + raise exception 'Finishing differs from its requirement'; + end if; + v:=v || jsonb_build_object('last_synced_at',clock_timestamp()); + insert into manufacturing.finishing as existing(production_key, color, required_quantity, active, requirement_id, last_synced_at) + select production_key, color, required_quantity, active, requirement_id, last_synced_at from jsonb_populate_record(null::manufacturing.finishing,v) + on conflict (production_key) where production_key is not null and production_key<>'' do update set + color=case when v ? 'color' then excluded.color else existing.color end, + required_quantity=case when v ? 'required_quantity' then excluded.required_quantity else existing.required_quantity end, + active=case when v ? 'active' then excluded.active else existing.active end, + requirement_id=case when v ? 'requirement_id' then excluded.requirement_id else existing.requirement_id end, + last_synced_at=case when v ? 'last_synced_at' then excluded.last_synced_at else existing.last_synced_at end, + updated_at=clock_timestamp(); + end loop; + + -- Source root, then unambiguous legacy key/assembly fallback. Never infer scope from a shared part. + update manufacturing.requirements r set active_in_bom=false, engineering_changed=true, + last_synced_at=clock_timestamp(), updated_at=clock_timestamp() + where r.id in (select r.id from manufacturing.requirements r + left join manufacturing.assemblies a on a.id=r.assembly_id where coalesce(nullif(r.source_root,''), case when cardinality(string_to_array(r.production_key,'|'))>=5 then split_part(r.production_key,'|',1) end, a.assembly_number)=any(roots)) + and r.active_in_bom is distinct from false + and not exists(select 1 from jsonb_array_elements(p_payload->'requirements') desired where desired->>'production_key'=r.production_key); + get diagnostics requirements_deactivated = row_count; + + update manufacturing.operations o set active_in_routing=false, updated_at=clock_timestamp() + from manufacturing.requirements r left join manufacturing.assemblies a on a.id=r.assembly_id + where o.requirement_id=r.id and coalesce(nullif(r.source_root,''), case when cardinality(string_to_array(r.production_key,'|'))>=5 then split_part(r.production_key,'|',1) end, a.assembly_number)=any(roots) + and o.active_in_routing is distinct from false + and not exists(select 1 from jsonb_array_elements(p_payload->'operations') desired where desired->>'operation_key'=o.operation_key); + get diagnostics operations_deactivated = row_count; + + update manufacturing.finishing f set active=false, last_synced_at=clock_timestamp(), updated_at=clock_timestamp() + from manufacturing.requirements r left join manufacturing.assemblies a on a.id=r.assembly_id + where f.requirement_id=r.id and coalesce(nullif(r.source_root,''), case when cardinality(string_to_array(r.production_key,'|'))>=5 then split_part(r.production_key,'|',1) end, a.assembly_number)=any(roots) + and f.active is distinct from false + and not exists(select 1 from jsonb_array_elements(p_payload->'finishing') desired where desired->>'production_key'=f.production_key); + get diagnostics finishing_deactivated = row_count; + + if coalesce((p_payload->>'discovery_complete')::boolean,false) and coalesce(p_payload->>'discovery_master','')<>'' then + update manufacturing.assemblies set integration_status='Discovered — Master Unreleased', updated_at=clock_timestamp() + where discovery_master=p_payload->>'discovery_master' and integration_status='Missing from Main — Review' + and exists(select 1 from jsonb_array_elements_text(p_payload->'discovered_roots') n where n=assembly_number); + update manufacturing.assemblies set integration_status='Missing from Main — Review', updated_at=clock_timestamp() + where discovery_master=p_payload->>'discovery_master' and not exists( + select 1 from jsonb_array_elements_text(p_payload->'discovered_roots') n where n=assembly_number); + end if; + + for g in select value from jsonb_array_elements(p_payload->'attachments') loop + if coalesce(g->>'kind','') not in ('drawing-pdf','step') or coalesce(g->>'export_key','') !~ '^[0-9a-f]{64}$' + or jsonb_typeof(g->'files') is distinct from 'array' or jsonb_array_length(g->'files')=0 then + raise exception 'Invalid attachment group'; + end if; + if not exists(select 1 from jsonb_array_elements(p_payload->'parts') p where p->>'part_number'=g->>'part_number') then + raise exception 'Attachment part is absent from payload'; + end if; + select id into strict part_id_value from manufacturing.parts where part_number=g->>'part_number'; + position_value:=0; + for f in select value from jsonb_array_elements(g->'files') loop + if coalesce(f->>'sha256','') !~ '^[0-9a-f]{64}$' + or f->>'storage_bucket' is distinct from 'manufacturing-files' + or f->>'storage_path' is distinct from 'sha256/' || substr(f->>'sha256',1,2) || '/' || + (f->>'sha256') || (case g->>'kind' when 'drawing-pdf' then '.pdf' else '.step' end) + or coalesce(f->>'source_url','') !~ '^https://' + or coalesce(f->>'original_name','')='' + or f->>'content_type' is distinct from (case g->>'kind' when 'drawing-pdf' then 'application/pdf' else 'application/step' end) + or (f->>'byte_size')::bigint is null or (f->>'byte_size')::bigint<0 + or (f->>'verified_at')::timestamptz is null or jsonb_typeof(f->'source_metadata') is distinct from 'object' then + raise exception 'Invalid verified attachment'; + end if; + -- The bucket must already be private, and the uploaded object must exist. + if not exists(select 1 from storage.buckets b join storage.objects o on o.bucket_id=b.id + where b.id='manufacturing-files' and not b.public and o.name=f->>'storage_path') then + raise exception 'Verified attachment object is missing or bucket is public'; + end if; + insert into manufacturing.attachments(part_id,kind,position,source_field_id,source_url,source_metadata, + original_name,content_type,byte_size,sha256,storage_bucket,storage_path,verified_at) + values(part_id_value,g->>'kind',position_value,null,f->>'source_url',f->'source_metadata', + f->>'original_name',f->>'content_type',(f->>'byte_size')::bigint,f->>'sha256',f->>'storage_bucket', + f->>'storage_path',(f->>'verified_at')::timestamptz) + on conflict(part_id,kind,position) do update set + source_field_id=null,source_url=excluded.source_url,source_metadata=excluded.source_metadata, + original_name=excluded.original_name,content_type=excluded.content_type,byte_size=excluded.byte_size, + sha256=excluded.sha256,storage_bucket=excluded.storage_bucket,storage_path=excluded.storage_path, + verified_at=excluded.verified_at; + position_value:=position_value+1; + end loop; + delete from manufacturing.attachments where part_id=part_id_value and kind=g->>'kind' and position>=position_value; + if g->>'kind'='drawing-pdf' then + update manufacturing.parts set drawing_export_key=g->>'export_key', drawing_files=g->'files' where id=part_id_value; + else + update manufacturing.parts set step_export_key=g->>'export_key', step_files=g->'files' where id=part_id_value; + end if; + end loop; + if duplicate_operations_preserved>0 then + p_payload:=jsonb_set(p_payload,'{warnings}',(p_payload->'warnings') || jsonb_build_array( + format('Preserved %s existing duplicate operation rows and their shop work',duplicate_operations_preserved))); + end if; + result_summary:=jsonb_build_object('status',case when jsonb_array_length(p_payload->'warnings')>0 then 'partial' else 'success' end, + 'committed',true,'warnings',p_payload->'warnings','source_rows',p_payload->'source_rows', + 'synced_roots',p_payload->'synced_roots','deactivated',requirements_deactivated, + 'operations_deactivated',operations_deactivated,'finishing_deactivated',finishing_deactivated, + 'duplicate_operations_preserved',duplicate_operations_preserved, + 'file_groups_uploaded',jsonb_array_length(p_payload->'attachments'),'file_groups_cached',p_payload->'file_groups_cached'); + exception when others then + get stacked diagnostics error_message = message_text; + result_summary:=jsonb_build_object('status','failed','committed',false,'error',error_message); + end; + update manufacturing.engineering_sync_runs set status=result_summary->>'status',summary=result_summary, + finished_at=clock_timestamp() where id=p_run_id; + return result_summary; +end; +$$; + +revoke all on function public.manufacturing_begin_engineering_sync(uuid,text) from public, anon, authenticated; +grant execute on function public.manufacturing_begin_engineering_sync(uuid,text) to service_role; +revoke all on function public.manufacturing_finish_engineering_sync(uuid,text,jsonb) from public, anon, authenticated; +grant execute on function public.manufacturing_finish_engineering_sync(uuid,text,jsonb) to service_role; +revoke all on function public.manufacturing_engineering_sync_state() from public, anon, authenticated; +grant execute on function public.manufacturing_engineering_sync_state() to service_role; +revoke all on function public.manufacturing_engineering_file_state(text[]) from public, anon, authenticated; +grant execute on function public.manufacturing_engineering_file_state(text[]) to service_role; +revoke all on function public.manufacturing_apply_engineering_sync(uuid,jsonb) from public, anon, authenticated; +grant execute on function public.manufacturing_apply_engineering_sync(uuid,jsonb) to service_role; + +commit; diff --git a/tests/engineering-sync.test.mjs b/tests/engineering-sync.test.mjs new file mode 100644 index 0000000..c67e2ec --- /dev/null +++ b/tests/engineering-sync.test.mjs @@ -0,0 +1,209 @@ +// In-memory PostgreSQL only. No HTTP, Onshape, Supabase or backend credentials. +// npm install --no-save --package-lock=false @electric-sql/pglite@0.3.14 +// node tests/engineering-sync.test.mjs +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +const { PGlite } = await import(process.env.PGLITE_MODULE || '@electric-sql/pglite'); +const db = new PGlite(); +const sql = async (text, values = []) => (await db.query(text, values)).rows; +const read = path => readFile(new URL(path, import.meta.url), 'utf8'); +await db.exec(` + create role anon; create role authenticated; create role service_role; + create schema frc190_baserow_stage; + create table frc190_baserow_stage.snapshots(id uuid primary key); + create schema storage; + create table storage.buckets(id text primary key, public boolean not null); + create table storage.objects(bucket_id text, name text); + insert into storage.buckets values('manufacturing-files',false); +`); +await db.exec(await read('./fixtures/contract-normalized.sql')); +await db.exec(await read('./fixtures/contract-attachments.sql')); +await db.exec(await read('../supabase/production/20260906_onshape_engineering_sync.sql')); + +const begin = async () => { + const id = randomUUID(); + await sql('select public.manufacturing_begin_engineering_sync($1,$2)', [id, 'https://example.test/run']); + return id; +}; +const apply = async (payload, id) => + (await sql('select public.manufacturing_apply_engineering_sync($1,$2) result', [id || await begin(), payload]))[0].result; +const snapshot = async () => { + const data = {}; + for (const table of ['assemblies','parts','requirements','operations','finishing','attachments']) + data[table] = await sql(`select * from manufacturing.${table} order by id`); + return data; +}; +function payload(root = 'A-ONE', revision = 'A') { + const key = `${root}|${revision}|${root}|P-ONE|default`; + return { + assemblies: [{assembly_number:root,subsystem_name:'Subsystem',active:true, + sync_schema_version:'supabase-engineering-v1',latest_released_revision:revision, + discovery_master:'https://example.test/master',integration_status:'Discovered — Master Unreleased'}], + parts: [{part_number:'P-ONE',name:'Plate',revision:'A',active:true}], + requirements: [{production_key:key,part_number:'P-ONE',assembly_number:root, + source_root:root,source_assembly_revision:revision,required_part_revision:'A', + configuration:'default',required_quantity:4,finishing:'Red',machine_op1:'Haas CNC',active_in_bom:true}], + operations: [{operation_key:`${key}|OP1`,production_key:key,operation_number:'OP1', + machine:'Haas CNC',work_type:'Manufacturing',active_in_routing:true}], + finishing: [{production_key:key,color:'Red',required_quantity:4,active:true}], + attachments: [], synced_roots:[root],discovered_roots:[root],discovery_master:'', + discovery_complete:true,cad_synced:false,warnings:[],source_rows:1,file_groups_cached:0, + }; +} + +// Exercise the API using its actual restricted role. +await db.exec('set role service_role'); +await assert.rejects(sql('select * from manufacturing.parts'), /permission denied/); +const first = payload(); +const initialResult = await apply(first); +assert.equal(initialResult.status, 'success', JSON.stringify(initialResult)); +await db.exec('reset role'); +const created = await snapshot(); +assert.equal(created.requirements[0].part_id, created.parts[0].id); +assert.equal(created.requirements[0].assembly_id, created.assemblies[0].id); +assert.equal(created.operations[0].requirement_id, created.requirements[0].id); +assert.equal(created.finishing[0].requirement_id, created.requirements[0].id); + +// Seed every shop-owned column from model.ts, plus normalized location fields. +const model = await read('./fixtures/contract-model.ts'); +const shopByTable = {}; +for (const section of model.split('{ name: ').slice(1)) { + const match = section.match(/^"(\w+)"/); + if (!match) continue; + const table = match[1]; + shopByTable[table] = [...section.matchAll(/\["(\w+)","[^"]+","(\w+)","shop"\]/g)] + .map(([, name, kind]) => [name, kind]); +} +await db.exec("insert into manufacturing.locations(location_key,name) values('L','Shelf')"); +for (const [table, fields] of Object.entries(shopByTable)) { + for (const [name, kind] of fields) { + const value = kind === 'number' ? 7 : kind === 'date' ? '2026-09-01T00:00:00Z' : 'Shop-owned sentinel'; + await sql(`update manufacturing.${table} set ${name}=$1`, [value]); + } + await db.exec(`update manufacturing.${table} set location_id=1`); +} +await db.exec(`update manufacturing.requirements set part_location='On Robot', + location_updated_by='shop-user',location_updated_at='2026-09-01T00:00:00Z'`); +await db.exec(`insert into manufacturing.operation_allocations + select id,1,'shop-user','Shop',4,2,'{"untouched":true}' from manufacturing.operations`); +const shopBefore = await snapshot(); +const allocations = await sql('select * from manufacturing.operation_allocations'); +first.parts[0].name = 'Revised engineering name'; +first.requirements[0].required_quantity = 9; +first.finishing[0].required_quantity = 9; +assert.equal((await apply(first)).status, 'success'); +const shopAfter = await snapshot(); +for (const [table, fields] of Object.entries(shopByTable)) { + for (const name of [...fields.map(([name])=>name),'location_id', + ...(table === 'requirements' ? ['part_location','location_updated_by','location_updated_at'] : [])]) + assert.deepEqual(shopAfter[table][0][name], shopBefore[table][0][name], `${table}.${name}`); +} +assert.deepEqual(await sql('select * from manufacturing.operation_allocations'), allocations); +assert.equal(shopAfter.requirements[0].engineering_changed,true); +assert.equal(shopAfter.parts[0].name,'Revised engineering name'); + +// Every shop column is rejected by the server as well as omitted by the client. +for (const [table, fields] of Object.entries(shopByTable)) { + for (const [field] of fields) { + const bad = structuredClone(first); bad[table][0][field]='forbidden'; + const before=await snapshot(); + assert.equal((await apply(bad)).status,'failed', `${table}.${field}`); + assert.deepEqual(await snapshot(),before); + } +} + +// A late FK failure must roll back earlier assembly/part upserts, including markers. +let bad=structuredClone(first); +bad.parts[0].name='MUST ROLL BACK'; +bad.operations[0].production_key='missing'; +let before=await snapshot(); +assert.equal((await apply(bad)).status,'failed'); +assert.deepEqual(await snapshot(),before); +assert.equal((await sql("select status from manufacturing.engineering_sync_runs order by started_at desc limit 1"))[0].status,'failed'); + +// Shared part across two roots: only the successfully synced root loses old rows. +assert.equal((await apply(payload('A-TWO'))).status,'success'); +const secondRootBefore = (await snapshot()); +const next=payload('A-ONE','B'); +next.warnings=['A-TWO could not be synced and was skipped']; +next.discovery_complete=false; +next.discovery_master='https://example.test/master'; +assert.equal((await apply(next)).status,'partial'); +const scoped=await snapshot(); +const oldRequirement=scoped.requirements.find(r=>r.production_key===first.requirements[0].production_key); +assert.equal(oldRequirement.active_in_bom,false); +assert.equal(scoped.operations.find(o=>o.requirement_id===oldRequirement.id).active_in_routing,false); +assert.equal(scoped.finishing.find(f=>f.requirement_id===oldRequirement.id).active,false); +for (const table of ['requirements','operations','finishing']) { + const other = secondRootBefore[table].at(-1); + assert.deepEqual(scoped[table].find(r=>r.id===other.id),other); +} +assert.equal(scoped.assemblies.find(a=>a.assembly_number==='A-TWO').integration_status,'Discovered — Master Unreleased'); + +// Membership-only reconciliation never mutates work, even for a missing root. +const membership=payload(); +for (const table of ['assemblies','parts','requirements','operations','finishing']) membership[table]=[]; +membership.synced_roots=[]; +membership.discovered_roots=['A-ONE']; +membership.discovery_master='https://example.test/master'; +let workBefore=await snapshot(); +assert.equal((await apply(membership)).status,'success'); +let workAfter=await snapshot(); +for (const table of ['parts','requirements','operations','finishing','attachments']) + assert.deepEqual(workAfter[table],workBefore[table]); +assert.equal(workAfter.assemblies.find(a=>a.assembly_number==='A-TWO').integration_status,'Missing from Main — Review'); +membership.discovered_roots=['A-ONE','A-TWO']; +assert.equal((await apply(membership)).status,'success'); +assert.equal((await snapshot()).assemblies.find(a=>a.assembly_number==='A-TWO').integration_status,'Discovered — Master Unreleased'); + +// Empty released BOM legitimately deactivates the successful root's prior work. +const empty=payload('A-ONE','C'); +for (const table of ['parts','requirements','operations','finishing']) empty[table]=[]; +assert.equal((await apply(empty)).status,'success'); +assert.equal((await sql("select count(*)::int n from manufacturing.requirements where source_root='A-ONE' and active_in_bom"))[0].n,0); +bad=structuredClone(empty);bad.synced_roots=[]; +assert.equal((await apply(bad)).status,'failed'); + +// Private attachment registration, replacement, retries and truncating obsolete positions. +const withFiles=payload('A-ONE','D'); +const file = digit => ({original_name:'plate.step',content_type:'application/step',byte_size:4, + sha256:digit.repeat(64),storage_bucket:'manufacturing-files',storage_path:`sha256/${digit.repeat(2)}/${digit.repeat(64)}.step`, + source_url:`https://cad.onshape.com/export/${digit}`,source_metadata:{export_key:digit},verified_at:'2026-09-06T00:00:00Z'}); +for (const digit of ['a','b','c']) await sql('insert into storage.objects values($1,$2)', ['manufacturing-files',file(digit).storage_path]); +withFiles.attachments=[{part_number:'P-ONE',kind:'step',export_key:'a'.repeat(64),files:[file('a'),file('b')]}]; +withFiles.cad_synced=true; +assert.equal((await apply(withFiles)).status,'success'); +const attachmentIds=(await snapshot()).attachments.map(a=>a.id); +assert.equal((await apply(withFiles)).status,'success'); +assert.deepEqual((await snapshot()).attachments.map(a=>a.id),attachmentIds); +withFiles.attachments[0]={part_number:'P-ONE',kind:'step',export_key:'c'.repeat(64),files:[file('c')]}; +assert.equal((await apply(withFiles)).status,'success'); +assert.equal((await snapshot()).attachments.length,1); +assert.equal((await snapshot()).attachments[0].sha256,'c'.repeat(64)); +const apiFile=(await sql("select public.manufacturing_file_for_requirement($1,'step') result",[(await snapshot()).requirements.at(-1).id]))[0].result; +assert.equal(apiFile.path,file('c').storage_path); +bad=structuredClone(withFiles);bad.attachments[0].files[0].storage_path='wrong'; +before=await snapshot();assert.equal((await apply(bad)).status,'failed');assert.deepEqual(await snapshot(),before); +await db.exec("update storage.buckets set public=true"); +assert.equal((await apply(withFiles)).status,'failed'); +await db.exec("update storage.buckets set public=false"); + +// Retrying an acknowledged or uncertain commit is idempotent; an older run cannot overwrite it. +const older=await begin(), newer=await begin(); +assert.equal((await apply(withFiles,newer)).status,'success'); +before=await snapshot(); +assert.equal((await apply(withFiles,newer)).status,'success'); +await sql("select public.manufacturing_finish_engineering_sync($1,'failed','{}')",[newer]); +assert.equal((await sql('select status from manufacturing.engineering_sync_runs where id=$1',[newer]))[0].status,'success'); +assert.equal((await apply(withFiles,older)).status,'failed'); +assert.deepEqual(await snapshot(),before); + +for (const role of ['anon','authenticated']) { + await db.exec(`set role ${role}`); + await assert.rejects(sql('select public.manufacturing_engineering_sync_state()'), /permission denied/); + await assert.rejects(sql('select public.manufacturing_apply_engineering_sync($1,$2)',[randomUUID(),first]), /permission denied/); + await db.exec('reset role'); +} +await db.close(); +console.log('PASS: exact contract migration, relationships, all shop fields, scoped deactivation, rollback, attachments, retries, concurrency and grants'); diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..c381ebb --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,16 @@ +These three files are exact, unmodified contract fixtures from +`coreylu2027/190-Manufacturing`, branch `experimental`, commit +`b915b1439ae4fed616f24935de552734ebea525b`: + +- `contract-normalized.sql`: `supabase/production/20260905_normalized_manufacturing.sql` +- `contract-attachments.sql`: `supabase/production/20260905_manufacturing_attachments.sql` +- `contract-model.ts`: `lib/manufacturing/model.ts` + +The local PostgreSQL test loads the schema and attachment fixtures before the +new migration, and extracts shop-owned fields from the model for preservation +and rejection tests. Legacy source identifiers in these fixtures are required +by the authoritative contract; they are not integration configuration. + +The test supplies minimal local staging and Storage tables. It never starts +Supabase or contacts a backend. Storage HTTP behavior is covered separately +with Python mocks. diff --git a/tests/fixtures/contract-attachments.sql b/tests/fixtures/contract-attachments.sql new file mode 100644 index 0000000..ea69f1c --- /dev/null +++ b/tests/fixtures/contract-attachments.sql @@ -0,0 +1,131 @@ +-- Private attachment catalog for binaries copied from Baserow to Supabase +-- Storage. Apply after the normalized manufacturing schema. +begin; + +create table manufacturing.attachments ( + id bigint generated by default as identity primary key, + part_id bigint not null references manufacturing.parts(id) on delete restrict, + kind text not null check (kind in ('drawing-pdf', 'step')), + position integer not null check (position >= 0), + source_field_id bigint not null, + source_url text not null, + source_metadata jsonb not null, + original_name text not null, + content_type text not null, + byte_size bigint not null check (byte_size >= 0), + sha256 text not null check (sha256 ~ '^[0-9a-f]{64}$'), + storage_bucket text not null, + storage_path text not null, + verified_at timestamptz not null, + created_at timestamptz not null default clock_timestamp(), + unique (part_id, kind, position), + unique (source_url) +); +create index attachments_part_kind_idx on manufacturing.attachments(part_id, kind, position); +alter table manufacturing.attachments enable row level security; +revoke all on manufacturing.attachments from public, anon, authenticated, service_role; +revoke all on sequence manufacturing.attachments_id_seq from public, anon, authenticated, service_role; + +create function public.manufacturing_register_attachment( + p_part_id bigint, + p_kind text, + p_position integer, + p_source_field_id bigint, + p_source_url text, + p_source_metadata jsonb, + p_original_name text, + p_content_type text, + p_byte_size bigint, + p_sha256 text, + p_storage_bucket text, + p_storage_path text, + p_verified_at timestamptz +) returns bigint +language plpgsql security definer set search_path = '' as $$ +declare existing manufacturing.attachments; attachment_id bigint; +begin + if p_kind not in ('drawing-pdf', 'step') or p_position < 0 + or p_source_url !~ '^https://' or coalesce(p_original_name, '') = '' + or coalesce(p_content_type, '') = '' or p_byte_size < 0 + or p_sha256 !~ '^[0-9a-f]{64}$' or p_storage_bucket <> 'manufacturing-files' + or p_storage_path !~ '^sha256/[0-9a-f]{2}/[0-9a-f]{64}\.(pdf|step)$' + or p_source_metadata is null or p_verified_at is null then + raise exception 'Invalid manufacturing attachment'; + end if; + if not exists(select 1 from manufacturing.parts where id = p_part_id) then + raise exception 'Attachment part is missing' using errcode = '23503'; + end if; + + select * into existing from manufacturing.attachments + where (part_id, kind, position) = (p_part_id, p_kind, p_position) + or source_url = p_source_url + for update; + if found then + if existing.part_id is distinct from p_part_id or existing.kind is distinct from p_kind + or existing.position is distinct from p_position or existing.source_field_id is distinct from p_source_field_id + or existing.source_url is distinct from p_source_url or existing.source_metadata is distinct from p_source_metadata + or existing.original_name is distinct from p_original_name or existing.content_type is distinct from p_content_type + or existing.byte_size is distinct from p_byte_size or existing.sha256 is distinct from p_sha256 + or existing.storage_bucket is distinct from p_storage_bucket or existing.storage_path is distinct from p_storage_path then + raise exception 'Attachment registration conflicts with verified data' using errcode = '40001'; + end if; + return existing.id; + end if; + + insert into manufacturing.attachments( + part_id, kind, position, source_field_id, source_url, source_metadata, + original_name, content_type, byte_size, sha256, storage_bucket, + storage_path, verified_at + ) values ( + p_part_id, p_kind, p_position, p_source_field_id, p_source_url, + p_source_metadata, p_original_name, p_content_type, p_byte_size, p_sha256, + p_storage_bucket, p_storage_path, p_verified_at + ) returning id into attachment_id; + return attachment_id; +end; +$$; +revoke all on function public.manufacturing_register_attachment(bigint,text,integer,bigint,text,jsonb,text,text,bigint,text,text,text,timestamptz) + from public, anon, authenticated; +grant execute on function public.manufacturing_register_attachment(bigint,text,integer,bigint,text,jsonb,text,text,bigint,text,text,text,timestamptz) + to service_role; + +create function public.manufacturing_file_for_requirement(p_requirement_id bigint, p_kind text) +returns jsonb language sql stable security definer set search_path = '' as $$ + select jsonb_build_object( + 'bucket', a.storage_bucket, + 'path', a.storage_path, + 'name', a.original_name, + 'content_type', a.content_type, + 'byte_size', a.byte_size, + 'sha256', a.sha256 + ) + from manufacturing.requirements r + join manufacturing.attachments a on a.part_id = r.part_id + where r.id = p_requirement_id and a.kind = p_kind + order by a.position, a.id + limit 1; +$$; +revoke all on function public.manufacturing_file_for_requirement(bigint,text) from public, anon, authenticated; +grant execute on function public.manufacturing_file_for_requirement(bigint,text) to service_role; + +create function public.manufacturing_attachment_manifest() +returns jsonb language sql stable security definer set search_path = '' as $$ + select coalesce(jsonb_agg(jsonb_build_object( + 'part_id', part_id, + 'kind', kind, + 'position', position, + 'source_field_id', source_field_id, + 'source_url', source_url, + 'original_name', original_name, + 'content_type', content_type, + 'byte_size', byte_size, + 'sha256', sha256, + 'storage_bucket', storage_bucket, + 'storage_path', storage_path + ) order by part_id, kind, position), '[]'::jsonb) + from manufacturing.attachments; +$$; +revoke all on function public.manufacturing_attachment_manifest() from public, anon, authenticated; +grant execute on function public.manufacturing_attachment_manifest() to service_role; + +commit; diff --git a/tests/fixtures/contract-model.ts b/tests/fixtures/contract-model.ts new file mode 100644 index 0000000..ceb6b0d --- /dev/null +++ b/tests/fixtures/contract-model.ts @@ -0,0 +1,98 @@ +export type RawRow = Record & { id: number }; +export interface ManufacturingAttachment { + partId: number; + kind: "drawing-pdf" | "step"; + position: number; + originalName: string; +} +export type Kind = "text" | "number" | "boolean" | "select" | "link" | "date" | "json"; +export type Column = readonly [name: string, source: string, kind: Kind, owner: "engineering" | "shop"]; +export interface Entity { name: string; tableId: number; key: string; columns: readonly Column[] } +export const ENTITIES: readonly Entity[] = [ + { name: "assemblies", tableId: 1119645, key: "assembly_number", columns: [ + ["assembly_number","Assembly Number","text","engineering"],["subsystem_name","Subsystem Name","text","engineering"], + ["active","Active","boolean","engineering"],["notes","Notes","text","shop"],["sync_schema_version","Sync Schema Version","text","engineering"] + ]}, + { name: "parts", tableId: 1119641, key: "part_number", columns: [ + ["part_number","Part Number","text","engineering"],["name","Name","text","engineering"],["description","Description","text","engineering"], + ["material","Material","text","engineering"],["manufacturing_method","Manufacturing Method","text","engineering"], + ["vendor","Vendor","text","engineering"],["revision","Revision","text","engineering"],["onshape_url","OnShape Text","text","engineering"], + ["category","Category","text","engineering"],["drawing_url","Onshape Drawing","text","engineering"], + ["drawing_files","Drawing PDF","json","engineering"],["step_files","STEP File","json","engineering"], + ["drawing_export_key","Drawing PDF Export Key","text","engineering"],["step_export_key","STEP Export Key","text","engineering"], + ["active","Active","boolean","engineering"],["last_synced_at","Last Synced At","date","engineering"] + ]}, + { name: "requirements", tableId: 1119642, key: "production_key", columns: [ + ["production_key","Production Key","text","engineering"],["part_id","Part","link","engineering"],["assembly_id","Assembly","link","engineering"], + ["configuration","Configuration","text","engineering"],["required_quantity","Required Quantity","number","engineering"], + ["bom_positions","BOM Positions","text","engineering"],["onshape_url","Onshape Source","text","engineering"], + ["source_document","Source Document","text","engineering"],["source_root","Source Root","text","engineering"], + ["source_assembly_revision","Source Assembly Revision","text","engineering"],["required_part_revision","Required Part Revision","text","engineering"], + ["machine_op1","Machine OP1","select","engineering"],["machine_op2","Machine OP2","select","engineering"], + ["machine_op3","Machine OP3","select","engineering"],["machine_op4","Machine OP4","select","engineering"], + ["finishing","Finishing","select","engineering"],["active_in_bom","Active in BOM","boolean","engineering"], + ["engineering_changed","Engineering Changed","boolean","engineering"],["last_synced_at","Last Synced At","date","engineering"], + ["status","Status","select","shop"],["machinist","Machinist","text","shop"],["qc_outcome","QC Outcome","select","shop"], + ["qc_notes","QC Notes","text","shop"],["qc_reviewed_by","QC Reviewed By","text","shop"],["qc_reviewed_at","QC Reviewed At","date","shop"], + ["disposition","Disposition","select","shop"] + ]}, + { name: "operations", tableId: 1169282, key: "operation_key", columns: [ + ["operation_key","Operation","text","engineering"],["requirement_id","Production Requirement","link","engineering"], + ["operation_number","Operation Number","select","engineering"],["machine","Machine","select","engineering"], + ["work_type","Work Type","select","engineering"],["active_in_routing","Active in Routing","boolean","engineering"], + ["status","Status","select","shop"],["machinist","Machinist","text","shop"],["started_at","Started At","date","shop"], + ["completed_at","Completed At","date","shop"],["claimed_quantity","Claimed Quantity","number","shop"], + ["completed_quantity","Completed Quantity","number","shop"],["quantity_ledger","Quantity Ledger","text","shop"], + ["cam_program_path","CAM Program Path","text","shop"],["cam_notes","CAM Notes","text","shop"] + ]}, + { name: "finishing", tableId: 1170619, key: "production_key", columns: [ + ["production_key","Production Key","text","engineering"],["requirement_id","Production Requirement","link","engineering"], + ["color","Powder Coat Color","select","engineering"],["required_quantity","Required Quantity","number","engineering"], + ["active","Active","boolean","engineering"],["last_synced_at","Last Synced At","date","engineering"], + ["machinist","Machinist","text","shop"] + ]} +]; +export type NormalizedRow = Record & { id: number; baserow_id: number | null; source_row: RawRow }; +export function fieldValue(value: unknown, kind: Kind): unknown { + if (value === undefined || value === null || value === "") return null; + if (kind === "select") return typeof value === "object" && "value" in value ? (value as {value: unknown}).value : value; + if (kind === "link") { + if (!Array.isArray(value) || value.length > 1) throw new Error("Expected zero or one link; refusing to collapse a relationship"); + return value.length ? (typeof value[0] === "object" ? value[0].id : value[0]) : null; + } + if (kind === "number") { + const number = Number(value); + if (!Number.isFinite(number)) throw new Error("Non-finite source quantity"); + return number; + } + return value; +} +export function normalizeRow(entity: Entity, row: RawRow) { + return { id: row.id, baserow_id: row.id, source_row: row, + ...Object.fromEntries(entity.columns.map(([column, source, kind]) => [column, fieldValue(row[source], kind)])) }; +} +function timestampKey(value: unknown) { + const text=String(value).replace(" ","T"); + const match=text.match(/^(.+?)(?:\.(\d+))?(Z|[+-]\d{2}:?\d{2})$/); + if(!match) return text; + const ms=Date.parse(match[1]+match[3]); + if(!Number.isFinite(ms)) return text; + return (BigInt(ms)*BigInt(1000000)+BigInt((match[2]??"").padEnd(9,"0"))).toString(); +} +function equalValue(a: unknown, b: unknown, kind: Kind) { + if (kind === "date" && a && b) return timestampKey(a) === timestampKey(b); + return JSON.stringify(a) === JSON.stringify(b); +} +// Normalized columns are authoritative. Keep source display metadata and exact +// timestamp strings when the normalized value has not changed. +export function denormalizeRow(entity: Entity, record: NormalizedRow): RawRow { + const result: RawRow = { ...record.source_row, id: record.id }; + for (const [column, source, kind] of entity.columns) { + const current = record[column] ?? null; + if (equalValue(fieldValue(result[source], kind), current, kind)) continue; + if (kind === "select") result[source] = current === null ? null : { ...(typeof result[source] === "object" ? result[source] as object : {}), value: current }; + else if (kind === "link") result[source] = current === null ? [] : [{ id: current, value: "" }]; + else result[source] = current; + } + return result; +} diff --git a/tests/fixtures/contract-normalized.sql b/tests/fixtures/contract-normalized.sql new file mode 100644 index 0000000..dcfa41a --- /dev/null +++ b/tests/fixtures/contract-normalized.sql @@ -0,0 +1,207 @@ +-- Additive production-candidate schema. Apply explicitly; no live cutover. +begin; +create schema manufacturing; +revoke all on schema manufacturing from public, anon, authenticated; +create table manufacturing.locations ( + id bigint generated by default as identity primary key, + baserow_id bigint unique, location_key text unique not null, + name text not null, description text, active boolean not null default true, + created_at timestamptz not null default now() +); +create table manufacturing.assemblies ( + id bigint generated by default as identity primary key, + baserow_id bigint unique, + source_snapshot_id uuid references frc190_baserow_stage.snapshots(id), + source_row jsonb not null default '{}'::jsonb, + location_id bigint references manufacturing.locations(id), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + assembly_number text, + subsystem_name text, + active boolean, + notes text, + sync_schema_version text, + check (baserow_id is null or baserow_id = id) +); +create index assemblies_key_idx on manufacturing.assemblies (assembly_number); +create unique index assemblies_key_unique on manufacturing.assemblies (assembly_number) where assembly_number is not null and assembly_number <> ''; +create table manufacturing.parts ( + id bigint generated by default as identity primary key, + baserow_id bigint unique, + source_snapshot_id uuid references frc190_baserow_stage.snapshots(id), + source_row jsonb not null default '{}'::jsonb, + location_id bigint references manufacturing.locations(id), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + part_number text, + name text, + description text, + material text, + manufacturing_method text, + vendor text, + revision text, + onshape_url text, + category text, + drawing_url text, + drawing_files jsonb, + step_files jsonb, + drawing_export_key text, + step_export_key text, + active boolean, + last_synced_at timestamptz, + check (baserow_id is null or baserow_id = id) +); +create index parts_key_idx on manufacturing.parts (part_number); +create unique index parts_key_unique on manufacturing.parts (part_number) where part_number is not null and part_number <> ''; +create table manufacturing.requirements ( + id bigint generated by default as identity primary key, + baserow_id bigint unique, + source_snapshot_id uuid references frc190_baserow_stage.snapshots(id), + source_row jsonb not null default '{}'::jsonb, + location_id bigint references manufacturing.locations(id), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + production_key text, + part_id bigint, + assembly_id bigint, + configuration text, + required_quantity numeric, + bom_positions text, + onshape_url text, + source_document text, + source_root text, + source_assembly_revision text, + required_part_revision text, + machine_op1 text, + machine_op2 text, + machine_op3 text, + machine_op4 text, + finishing text, + active_in_bom boolean, + engineering_changed boolean, + last_synced_at timestamptz, + status text, + machinist text, + qc_outcome text, + qc_notes text, + qc_reviewed_by text, + qc_reviewed_at timestamptz, + disposition text, + part_location text, + location_updated_by text, + location_updated_at timestamptz, + constraint requirements_part_location_check check (part_location is null or part_location in ( + 'Clarke 1','Clarke 2','Clarke 3','Clarke 4','Clarke 5','Clarke 6','Clarke 7','Clarke 8', + 'Kwolek 1-1','Kwolek 1-2','Kwolek 1-3','Kwolek 1-4','Kwolek 1-5','Kwolek 1-6','Kwolek 1-7','Kwolek 1-8', + 'Kwolek 2-1','Kwolek 2-2','Kwolek 2-3','Kwolek 2-4','Kwolek 2-5','Kwolek 2-6','Kwolek 2-7','Kwolek 2-8', + 'Hopper 1','Hopper 2','Hopper 3','Hopper 4','Hopper 5','Hopper 6','Hopper 7','Hopper 8', + 'Jemison 1-1','Jemison 1-2','Jemison 1-3','Jemison 1-4','Jemison 1-5','Jemison 1-6','Jemison 1-7','Jemison 1-8', + 'Jemison 2-1','Jemison 2-2','Jemison 2-3','Jemison 2-4','Jemison 2-5','Jemison 2-6','Jemison 2-7','Jemison 2-8', + 'Shelf 1','Shelf 2','Shelf 3','On Robot' + )), + constraint requirements_location_attribution_check check ( + (location_updated_by is null) = (location_updated_at is null) + ), + check (baserow_id is null or baserow_id = id) +); +create index requirements_key_idx on manufacturing.requirements (production_key); +create unique index requirements_key_unique on manufacturing.requirements (production_key) where production_key is not null and production_key <> ''; +create table manufacturing.operations ( + id bigint generated by default as identity primary key, + baserow_id bigint unique, + source_snapshot_id uuid references frc190_baserow_stage.snapshots(id), + source_row jsonb not null default '{}'::jsonb, + location_id bigint references manufacturing.locations(id), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + operation_key text, + requirement_id bigint, + operation_number text, + machine text, + work_type text, + active_in_routing boolean, + status text, + machinist text, + started_at timestamptz, + completed_at timestamptz, + claimed_quantity numeric, + completed_quantity numeric, + quantity_ledger text, + cam_program_path text, + cam_notes text, + check (baserow_id is null or baserow_id = id) +); +create index operations_key_idx on manufacturing.operations (operation_key); +create table manufacturing.finishing ( + id bigint generated by default as identity primary key, + baserow_id bigint unique, + source_snapshot_id uuid references frc190_baserow_stage.snapshots(id), + source_row jsonb not null default '{}'::jsonb, + location_id bigint references manufacturing.locations(id), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + production_key text, + requirement_id bigint, + color text, + required_quantity numeric, + active boolean, + last_synced_at timestamptz, + machinist text, + check (baserow_id is null or baserow_id = id) +); +create index finishing_key_idx on manufacturing.finishing (production_key); +create unique index finishing_key_unique on manufacturing.finishing (production_key) where production_key is not null and production_key <> ''; + +alter table manufacturing.requirements add foreign key (part_id) references manufacturing.parts(id), + add foreign key (assembly_id) references manufacturing.assemblies(id); +alter table manufacturing.operations add foreign key (requirement_id) references manufacturing.requirements(id); +alter table manufacturing.finishing add foreign key (requirement_id) references manufacturing.requirements(id); +create index operations_requirement_idx on manufacturing.operations(requirement_id); +create index finishing_requirement_idx on manufacturing.finishing(requirement_id); +create table manufacturing.operation_allocations ( + operation_id bigint not null references manufacturing.operations(id), + ordinal integer not null, + user_id text not null, display_name text not null, + claimed numeric not null, completed numeric not null, + source_allocation jsonb not null, + primary key (operation_id, ordinal) +); +comment on table manufacturing.operation_allocations is 'Imported ledger entries retain original identities, including legacy assignments; never resolved by display name'; +create table manufacturing.imports ( + snapshot_id uuid primary key references frc190_baserow_stage.snapshots(id), + imported_at timestamptz not null default now(), counts jsonb not null +); +alter table manufacturing.locations enable row level security; +revoke all on manufacturing.locations from public, anon, authenticated, service_role; +alter table manufacturing.assemblies enable row level security; +revoke all on manufacturing.assemblies from public, anon, authenticated, service_role; +alter table manufacturing.parts enable row level security; +revoke all on manufacturing.parts from public, anon, authenticated, service_role; +alter table manufacturing.requirements enable row level security; +revoke all on manufacturing.requirements from public, anon, authenticated, service_role; +alter table manufacturing.operations enable row level security; +revoke all on manufacturing.operations from public, anon, authenticated, service_role; +alter table manufacturing.finishing enable row level security; +revoke all on manufacturing.finishing from public, anon, authenticated, service_role; +alter table manufacturing.operation_allocations enable row level security; +revoke all on manufacturing.operation_allocations from public, anon, authenticated, service_role; +alter table manufacturing.imports enable row level security; +revoke all on manufacturing.imports from public, anon, authenticated, service_role; + +-- No schema exposure/config change. Server-side reads use this bounded RPC. +create function public.manufacturing_read_entity(p_entity text, p_offset integer default 0, p_limit integer default 500) +returns jsonb language plpgsql stable security definer set search_path = '' as $read$ +declare result jsonb; +begin + if p_entity not in ('assemblies','parts','requirements','operations','finishing','locations') + or p_offset < 0 or p_limit < 1 or p_limit > 500 then raise exception 'Invalid manufacturing page'; end if; + execute format('select jsonb_build_object(''rows'', coalesce((select jsonb_agg(to_jsonb(r)) from + (select * from manufacturing.%I order by id offset $1 limit $2) r), ''[]''::jsonb), + ''total'', (select count(*) from manufacturing.%I))', p_entity, p_entity) + into result using p_offset, p_limit; + return result; +end; +$read$; +revoke all on function public.manufacturing_read_entity(text,integer,integer) from public, anon, authenticated; +grant execute on function public.manufacturing_read_entity(text,integer,integer) to service_role; +commit; diff --git a/tests/imported-sync.test.mjs b/tests/imported-sync.test.mjs new file mode 100644 index 0000000..57f38f5 --- /dev/null +++ b/tests/imported-sync.test.mjs @@ -0,0 +1,81 @@ +// Regression for the read-only production preflight: imported duplicate operation +// keys and empty normalized revision fields. All data below is synthetic. +import assert from 'node:assert/strict'; +import {readFile} from 'node:fs/promises'; +import {randomUUID} from 'node:crypto'; +const {PGlite}=await import(process.env.PGLITE_MODULE || '@electric-sql/pglite'); +const db=new PGlite(); +const query=async (q,p=[]) => (await db.query(q,p)).rows; +const read=p=>readFile(new URL(p,import.meta.url),'utf8'); +await db.exec(`create role anon;create role authenticated;create role service_role; + create schema frc190_baserow_stage; + create table frc190_baserow_stage.snapshots(id uuid primary key); + create schema storage; + create table storage.buckets(id text primary key, public boolean not null); + create table storage.objects(bucket_id text,name text);`); +await db.exec(await read('./fixtures/contract-normalized.sql')); +await db.exec(await read('./fixtures/contract-attachments.sql')); +await db.exec(` + insert into manufacturing.assemblies(assembly_number,sync_schema_version) values('A-IMPORTED','source-document-v1'); + insert into manufacturing.parts(part_number,name,revision,step_export_key) values('P-IMPORTED','Plate','A','previous-export'); + insert into manufacturing.requirements(production_key,part_id,assembly_id,required_quantity,active_in_bom,status) + select 'A-IMPORTED|A|A-IMPORTED|P-IMPORTED|default',p.id,a.id,4,true,'In Progress' + from manufacturing.parts p cross join manufacturing.assemblies a; + insert into manufacturing.operations(operation_key,requirement_id,operation_number,machine,work_type, + active_in_routing,status,claimed_quantity,completed_quantity,quantity_ledger) + select r.production_key||'|OP1',r.id,'OP1','Haas CNC','Manufacturing',true, + case n when 1 then 'In Progress' else 'Complete' end,n,2*n,'separate ledger '||n + from manufacturing.requirements r cross join generate_series(1,2) n; + insert into manufacturing.operation_allocations(operation_id,ordinal,user_id,display_name,claimed,completed,source_allocation) + select id,1,'test-user-'||id,'Synthetic user',claimed_quantity,completed_quantity,'{}' + from manufacturing.operations; +`); +// The migration must apply to duplicates already present, without rewriting them. +const before=await query('select * from manufacturing.operations order by id'); +const allocations=await query('select * from manufacturing.operation_allocations order by operation_id'); +await db.exec(await read('../supabase/production/20260906_onshape_engineering_sync.sql')); +assert.deepEqual(await query('select * from manufacturing.operations order by id'),before); +const key='A-IMPORTED|A|A-IMPORTED|P-IMPORTED|default'; +const payload={ + assemblies:[{assembly_number:'A-IMPORTED',latest_released_revision:'A',sync_schema_version:'supabase-engineering-v1',active:true}], + parts:[{part_number:'P-IMPORTED',name:'Plate',revision:'A',active:true}], + requirements:[{production_key:key,part_number:'P-IMPORTED',assembly_number:'A-IMPORTED',source_root:'A-IMPORTED', + source_assembly_revision:'A',required_part_revision:'A',configuration:'default',required_quantity:4,active_in_bom:true}], + operations:[{operation_key:key+'|OP1',production_key:key,operation_number:'OP1',machine:'Haas CNC',work_type:'Manufacturing',active_in_routing:true}], + finishing:[],attachments:[],warnings:[],synced_roots:['A-IMPORTED'],discovered_roots:['A-IMPORTED'], + discovery_master:'',discovery_complete:true,cad_synced:false,source_rows:1,file_groups_cached:0, +}; +const apply=async p=>{ + const id=randomUUID(); + await query("select public.manufacturing_begin_engineering_sync($1,'local-test')",[id]); + return (await query('select public.manufacturing_apply_engineering_sync($1,$2) result',[id,p]))[0].result; +}; +const shopFields=['id','status','machinist','started_at','completed_at','claimed_quantity','completed_quantity', + 'quantity_ledger','cam_program_path','cam_notes','location_id']; +for(let pass=0;pass<2;pass++) { + const result=await apply(payload); + assert.equal(result.status,'partial',JSON.stringify(result)); + assert.equal(result.duplicate_operations_preserved,1); + const after=await query('select * from manufacturing.operations order by id'); + assert.equal(after.length,2); + for(let i=0;i<2;i++) for(const f of shopFields) assert.deepEqual(after[i][f],before[i][f],f); + assert.deepEqual(await query('select * from manufacturing.operation_allocations order by operation_id'),allocations); + const req=await query('select * from manufacturing.requirements'); + assert.equal(req.length,1);assert.equal(req[0].source_root,'A-IMPORTED');assert.equal(req[0].status,'In Progress'); + assert.equal((await query('select step_export_key from manufacturing.parts'))[0].step_export_key,'previous-export'); +} +const state=(await query('select public.manufacturing_engineering_sync_state() result'))[0].result; +assert.equal(state[0]['Latest Released Revision'],'A'); +assert.equal(state[0]['Sync Schema Version'],'supabase-engineering-v1'); +assert.equal(state[0]['CAD Synced'],false); + +// A key collision pointing at different shop work must roll the whole sync back. +await db.exec("update manufacturing.operations set work_type='CAM' where id=(select max(id) from manufacturing.operations)"); +const partsBefore=await query('select * from manufacturing.parts'); +const opsBefore=await query('select * from manufacturing.operations order by id'); +const bad=structuredClone(payload);bad.parts[0].name='must roll back'; +assert.equal((await apply(bad)).status,'failed'); +assert.deepEqual(await query('select * from manufacturing.parts'),partsBefore); +assert.deepEqual(await query('select * from manufacturing.operations order by id'),opsBefore); +await db.close(); +console.log('PASS: imported duplicates survive migration/replay, shop allocations stay intact, missing revision fields are populated, conflicting work rolls back');